{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.5714285714285714, "eval_steps": 500, "global_step": 500, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_fraction": 0.0, "completion_length": 2571.2083587646484, "epoch": 0.001142857142857143, "grad_norm": 0.2146759331226349, "kl": 0.0, "lambda_div_used": 0.6179041713476181, "learning_rate": 0.0, "loss": 0.0867, "reward": -0.10299695748835802, "reward_after_mean": -0.10299695748835802, "reward_after_std": 0.564858466386795, "reward_before_mean": 0.21363236638717353, "reward_before_std": 0.541789973154664, "reward_change_max": 0.0, "reward_change_mean": -0.31662931852042675, "reward_change_min": -0.5466086529195309, "reward_change_std": 0.20701106544584036, "reward_std": 0.5648584887385368, "rewards/accuracy_reward": 0.22916667349636555, "rewards/cosine_scaled_reward": -0.015534311532974243, "step": 1 }, { "clip_fraction": 0.0, "completion_length": 2804.395881652832, "epoch": 0.002285714285714286, "grad_norm": 0.21154765784740448, "kl": 0.0, "lambda_div_used": 0.5548827201128006, "learning_rate": 2e-08, "loss": 0.0255, "reward": -0.2257593870162964, "reward_after_mean": -0.2257593870162964, "reward_after_std": 0.34127857722342014, "reward_before_mean": 0.179365461692214, "reward_before_std": 0.2432677550241351, "reward_change_max": 0.0, "reward_change_mean": -0.4051248449832201, "reward_change_min": -0.5814888626337051, "reward_change_std": 0.22178608737885952, "reward_std": 0.34127858839929104, "rewards/accuracy_reward": 0.2291666716337204, "rewards/cosine_scaled_reward": -0.04980122856795788, "step": 2 }, { "clip_fraction": 0.0, "completion_length": 3336.8125, "epoch": 0.0034285714285714284, "grad_norm": 0.18279071152210236, "kl": 4.971027374267578e-05, "lambda_div_used": 0.5844422951340675, "learning_rate": 4e-08, "loss": -0.0326, "reward": -0.3244950734078884, "reward_after_mean": -0.3244950734078884, "reward_after_std": 0.43532974645495415, "reward_before_mean": -0.051717307418584824, "reward_before_std": 0.39013167656958103, "reward_change_max": 0.0, "reward_change_mean": -0.2727777659893036, "reward_change_min": -0.4898175373673439, "reward_change_std": 0.17431269027292728, "reward_std": 0.43532975018024445, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.13505065068602562, "step": 3 }, { "clip_fraction": 0.0, "completion_length": 2249.104202270508, "epoch": 0.004571428571428572, "grad_norm": 0.23296239972114563, "kl": 3.1284987926483154e-05, "lambda_div_used": 0.6461797505617142, "learning_rate": 6e-08, "loss": 0.0195, "reward": 0.032234033569693565, "reward_after_mean": 0.032234033569693565, "reward_after_std": 0.6864162627607584, "reward_before_mean": 0.350237637758255, "reward_before_std": 0.67749391682446, "reward_change_max": 0.0, "reward_change_mean": -0.31800360418856144, "reward_change_min": -0.5923457369208336, "reward_change_std": 0.21867174468934536, "reward_std": 0.686416283249855, "rewards/accuracy_reward": 0.27083334140479565, "rewards/cosine_scaled_reward": 0.07940429821610451, "step": 4 }, { "clip_fraction": 0.0, "completion_length": 3291.7708740234375, "epoch": 0.005714285714285714, "grad_norm": 0.1753876954317093, "kl": 4.207901656627655e-05, "lambda_div_used": 0.5958683490753174, "learning_rate": 8e-08, "loss": -0.0186, "reward": -0.3725212914869189, "reward_after_mean": -0.3725212914869189, "reward_after_std": 0.5095969215035439, "reward_before_mean": -0.14527345914393663, "reward_before_std": 0.4354726802557707, "reward_change_max": 0.0, "reward_change_mean": -0.22724783793091774, "reward_change_min": -0.34927351027727127, "reward_change_std": 0.12457231804728508, "reward_std": 0.5095969308167696, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.20777346193790436, "step": 5 }, { "clip_fraction": 0.0, "completion_length": 3081.625030517578, "epoch": 0.006857142857142857, "grad_norm": 0.18337981402873993, "kl": 4.323013126850128e-05, "lambda_div_used": 0.573668859899044, "learning_rate": 1e-07, "loss": 0.0112, "reward": -0.37891958840191364, "reward_after_mean": -0.37891958840191364, "reward_after_std": 0.40358329750597477, "reward_before_mean": -0.11088503152132034, "reward_before_std": 0.33515767380595207, "reward_change_max": 0.0, "reward_change_mean": -0.2680345606058836, "reward_change_min": -0.4288274832069874, "reward_change_std": 0.15655256435275078, "reward_std": 0.40358331240713596, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.17338503617793322, "step": 6 }, { "clip_fraction": 0.0, "completion_length": 3102.5000915527344, "epoch": 0.008, "grad_norm": 0.1660555601119995, "kl": 3.0435621738433838e-05, "lambda_div_used": 0.5801765397191048, "learning_rate": 1.2e-07, "loss": -0.0082, "reward": -0.23909527622163296, "reward_after_mean": -0.23909527622163296, "reward_after_std": 0.4915014058351517, "reward_before_mean": 0.11409430578351021, "reward_before_std": 0.3677805494517088, "reward_change_max": 0.0, "reward_change_mean": -0.3531895875930786, "reward_change_min": -0.535370722413063, "reward_change_std": 0.19709791243076324, "reward_std": 0.4915014076977968, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.07340569080406567, "step": 7 }, { "clip_fraction": 0.0, "completion_length": 2597.979202270508, "epoch": 0.009142857142857144, "grad_norm": 0.20249120891094208, "kl": 2.483278512954712e-05, "lambda_div_used": 0.6003148779273033, "learning_rate": 1.4e-07, "loss": -0.082, "reward": 0.16203208826482296, "reward_after_mean": 0.16203208826482296, "reward_after_std": 0.6564908493310213, "reward_before_mean": 0.7107675401493907, "reward_before_std": 0.461323918774724, "reward_change_max": 0.0, "reward_change_mean": -0.5487354453653097, "reward_change_min": -0.7694363072514534, "reward_change_std": 0.2932443069294095, "reward_std": 0.656490858644247, "rewards/accuracy_reward": 0.4583333358168602, "rewards/cosine_scaled_reward": 0.2524342043325305, "step": 8 }, { "clip_fraction": 0.0, "completion_length": 3232.166717529297, "epoch": 0.010285714285714285, "grad_norm": 0.1599472463130951, "kl": 4.266202449798584e-05, "lambda_div_used": 0.5893354639410973, "learning_rate": 1.6e-07, "loss": 0.0342, "reward": -0.3414417468011379, "reward_after_mean": -0.3414417468011379, "reward_after_std": 0.4444474130868912, "reward_before_mean": -0.07577274367213249, "reward_before_std": 0.41174139082431793, "reward_change_max": 0.0, "reward_change_mean": -0.2656690161675215, "reward_change_min": -0.4962993934750557, "reward_change_std": 0.17574716545641422, "reward_std": 0.4444474149495363, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.15910607110708952, "step": 9 }, { "clip_fraction": 0.0, "completion_length": 2778.3958435058594, "epoch": 0.011428571428571429, "grad_norm": 0.1842634677886963, "kl": 3.0197203159332275e-05, "lambda_div_used": 0.5796822905540466, "learning_rate": 1.8e-07, "loss": 0.0075, "reward": -0.32217366620898247, "reward_after_mean": -0.32217366620898247, "reward_after_std": 0.4148254282772541, "reward_before_mean": -0.0384827577508986, "reward_before_std": 0.3622512873262167, "reward_change_max": 0.0, "reward_change_mean": -0.2836909107863903, "reward_change_min": -0.4418584108352661, "reward_change_std": 0.1667974703013897, "reward_std": 0.41482544504106045, "rewards/accuracy_reward": 0.10416666977107525, "rewards/cosine_scaled_reward": -0.14264942612499, "step": 10 }, { "clip_fraction": 0.0, "completion_length": 3328.8541870117188, "epoch": 0.012571428571428572, "grad_norm": 0.14712053537368774, "kl": 2.7470290660858154e-05, "lambda_div_used": 0.5545761585235596, "learning_rate": 2e-07, "loss": 0.0247, "reward": -0.4563899512140779, "reward_after_mean": -0.4563899512140779, "reward_after_std": 0.3279168829321861, "reward_before_mean": -0.19584861118346453, "reward_before_std": 0.24491036403924227, "reward_change_max": 0.0, "reward_change_mean": -0.2605413384735584, "reward_change_min": -0.3718484900891781, "reward_change_std": 0.137719439342618, "reward_std": 0.3279168922454119, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.2166819479316473, "step": 11 }, { "clip_fraction": 0.0, "completion_length": 2589.3125610351562, "epoch": 0.013714285714285714, "grad_norm": 0.18253953754901886, "kl": 4.75030392408371e-05, "lambda_div_used": 0.580904982984066, "learning_rate": 2.1999999999999998e-07, "loss": 0.0132, "reward": -0.31201416440308094, "reward_after_mean": -0.31201416440308094, "reward_after_std": 0.428743164986372, "reward_before_mean": -0.026531244977377355, "reward_before_std": 0.3703338522464037, "reward_change_max": 0.0, "reward_change_mean": -0.2854829151183367, "reward_change_min": -0.4420909509062767, "reward_change_std": 0.16634919866919518, "reward_std": 0.4287431761622429, "rewards/accuracy_reward": 0.10416666977107525, "rewards/cosine_scaled_reward": -0.13069792464375496, "step": 12 }, { "clip_fraction": 0.0, "completion_length": 2627.750015258789, "epoch": 0.014857142857142857, "grad_norm": 0.171514630317688, "kl": 2.6579946279525757e-05, "lambda_div_used": 0.6024352535605431, "learning_rate": 2.4e-07, "loss": -0.0262, "reward": -0.16530478885397315, "reward_after_mean": -0.16530478885397315, "reward_after_std": 0.5007271338254213, "reward_before_mean": 0.13929638639092445, "reward_before_std": 0.47216523345559835, "reward_change_max": 0.0, "reward_change_mean": -0.3046011757105589, "reward_change_min": -0.48859554529190063, "reward_change_std": 0.18980247620493174, "reward_std": 0.5007271375507116, "rewards/accuracy_reward": 0.18750000558793545, "rewards/cosine_scaled_reward": -0.04820362292230129, "step": 13 }, { "clip_fraction": 0.0, "completion_length": 2903.104217529297, "epoch": 0.016, "grad_norm": 0.17632441222667694, "kl": 2.6632100343704224e-05, "lambda_div_used": 0.6022606194019318, "learning_rate": 2.6e-07, "loss": -0.02, "reward": -0.33581143367337063, "reward_after_mean": -0.33581143367337063, "reward_after_std": 0.5302506685256958, "reward_before_mean": -0.10058278171345592, "reward_before_std": 0.4697431940585375, "reward_change_max": 0.0, "reward_change_mean": -0.23522865399718285, "reward_change_min": -0.3982113152742386, "reward_change_std": 0.13983092363923788, "reward_std": 0.5302506759762764, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.1839161105453968, "step": 14 }, { "clip_fraction": 0.0, "completion_length": 2690.2708740234375, "epoch": 0.017142857142857144, "grad_norm": 0.19580987095832825, "kl": 2.584978938102722e-05, "lambda_div_used": 0.5800291448831558, "learning_rate": 2.8e-07, "loss": -0.0187, "reward": -0.18931277468800545, "reward_after_mean": -0.18931277468800545, "reward_after_std": 0.38942739367485046, "reward_before_mean": 0.14439615607261658, "reward_before_std": 0.3626243360340595, "reward_change_max": 0.0, "reward_change_mean": -0.3337089382112026, "reward_change_min": -0.5154752880334854, "reward_change_std": 0.2009873315691948, "reward_std": 0.38942740112543106, "rewards/accuracy_reward": 0.2083333432674408, "rewards/cosine_scaled_reward": -0.06393716484308243, "step": 15 }, { "clip_fraction": 0.0, "completion_length": 3572.6458435058594, "epoch": 0.018285714285714287, "grad_norm": 0.165513813495636, "kl": 4.07099723815918e-05, "lambda_div_used": 0.577268660068512, "learning_rate": 3e-07, "loss": 0.0054, "reward": -0.3996182642877102, "reward_after_mean": -0.3996182642877102, "reward_after_std": 0.4201182946562767, "reward_before_mean": -0.14905199501663446, "reward_before_std": 0.35537218395620584, "reward_change_max": 0.0, "reward_change_mean": -0.25056627579033375, "reward_change_min": -0.4106076806783676, "reward_change_std": 0.14937943872064352, "reward_std": 0.4201183207333088, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.21155200095381588, "step": 16 }, { "clip_fraction": 0.0, "completion_length": 2331.0625381469727, "epoch": 0.019428571428571427, "grad_norm": 0.28552213311195374, "kl": 4.392862319946289e-05, "lambda_div_used": 0.6344663128256798, "learning_rate": 3.2e-07, "loss": 0.1308, "reward": -0.1244519567117095, "reward_after_mean": -0.1244519567117095, "reward_after_std": 0.6570038888603449, "reward_before_mean": 0.1548405447974801, "reward_before_std": 0.6229406604543328, "reward_change_max": 0.0, "reward_change_mean": -0.27929250709712505, "reward_change_min": -0.5091209039092064, "reward_change_std": 0.187502876855433, "reward_std": 0.6570039205253124, "rewards/accuracy_reward": 0.1875000037252903, "rewards/cosine_scaled_reward": -0.03265946079045534, "step": 17 }, { "clip_fraction": 0.0, "completion_length": 2765.437515258789, "epoch": 0.02057142857142857, "grad_norm": 0.16589123010635376, "kl": 3.147125244140625e-05, "lambda_div_used": 0.6141805946826935, "learning_rate": 3.4000000000000003e-07, "loss": 0.0601, "reward": -0.20549198891967535, "reward_after_mean": -0.20549198891967535, "reward_after_std": 0.579927084967494, "reward_before_mean": 0.06148822698742151, "reward_before_std": 0.5241483375430107, "reward_change_max": 0.0, "reward_change_mean": -0.2669802140444517, "reward_change_min": -0.406601931899786, "reward_change_std": 0.15793233551084995, "reward_std": 0.5799271017313004, "rewards/accuracy_reward": 0.14583333767950535, "rewards/cosine_scaled_reward": -0.08434510324150324, "step": 18 }, { "clip_fraction": 0.0, "completion_length": 2939.3541870117188, "epoch": 0.021714285714285714, "grad_norm": 0.1630236953496933, "kl": 2.981722354888916e-05, "lambda_div_used": 0.6274994388222694, "learning_rate": 3.6e-07, "loss": 0.0239, "reward": 0.0620431462302804, "reward_after_mean": 0.0620431462302804, "reward_after_std": 0.6165293771773577, "reward_before_mean": 0.43255577981472015, "reward_before_std": 0.5838612085208297, "reward_change_max": 0.0, "reward_change_mean": -0.3705126289278269, "reward_change_min": -0.5637077167630196, "reward_change_std": 0.2224423261359334, "reward_std": 0.6165293958038092, "rewards/accuracy_reward": 0.3541666828095913, "rewards/cosine_scaled_reward": 0.07838911190629005, "step": 19 }, { "clip_fraction": 0.0, "completion_length": 2363.500011444092, "epoch": 0.022857142857142857, "grad_norm": 0.224549800157547, "kl": 1.7490237951278687e-05, "lambda_div_used": 0.6053463593125343, "learning_rate": 3.7999999999999996e-07, "loss": 0.1153, "reward": -0.13633326813578606, "reward_after_mean": -0.13633326813578606, "reward_after_std": 0.5765859521925449, "reward_before_mean": 0.20912488270550966, "reward_before_std": 0.49052994698286057, "reward_change_max": 0.0, "reward_change_mean": -0.3454581666737795, "reward_change_min": -0.5730844996869564, "reward_change_std": 0.21382273454219103, "reward_std": 0.5765859596431255, "rewards/accuracy_reward": 0.25000000186264515, "rewards/cosine_scaled_reward": -0.04087511729449034, "step": 20 }, { "clip_fraction": 0.0, "completion_length": 2793.0833892822266, "epoch": 0.024, "grad_norm": 0.1792718768119812, "kl": 3.4965574741363525e-05, "lambda_div_used": 0.5907231569290161, "learning_rate": 4e-07, "loss": 0.0123, "reward": -0.26306698843836784, "reward_after_mean": -0.26306698843836784, "reward_after_std": 0.48109759390354156, "reward_before_mean": 0.025504212244413793, "reward_before_std": 0.41451304592192173, "reward_change_max": 0.0, "reward_change_mean": -0.2885712068527937, "reward_change_min": -0.42935387045145035, "reward_change_std": 0.1654765186831355, "reward_std": 0.4810976069420576, "rewards/accuracy_reward": 0.1250000037252903, "rewards/cosine_scaled_reward": -0.0994957908987999, "step": 21 }, { "clip_fraction": 0.0, "completion_length": 1815.7916946411133, "epoch": 0.025142857142857144, "grad_norm": 0.2872166633605957, "kl": 3.246963024139404e-05, "lambda_div_used": 0.5785460472106934, "learning_rate": 4.1999999999999995e-07, "loss": -0.029, "reward": -0.17584524676203728, "reward_after_mean": -0.17584524676203728, "reward_after_std": 0.4682602845132351, "reward_before_mean": 0.19290187023580074, "reward_before_std": 0.35657503083348274, "reward_change_max": 0.0, "reward_change_mean": -0.36874712631106377, "reward_change_min": -0.5200564563274384, "reward_change_std": 0.1989120151847601, "reward_std": 0.46826029755175114, "rewards/accuracy_reward": 0.25000000558793545, "rewards/cosine_scaled_reward": -0.057098137214779854, "step": 22 }, { "clip_fraction": 0.0, "completion_length": 2543.916702270508, "epoch": 0.026285714285714287, "grad_norm": 0.19506987929344177, "kl": 3.40193510055542e-05, "lambda_div_used": 0.6334944739937782, "learning_rate": 4.3999999999999997e-07, "loss": 0.0732, "reward": -0.2326198872178793, "reward_after_mean": -0.2326198872178793, "reward_after_std": 0.6767921093851328, "reward_before_mean": -0.006950528710149229, "reward_before_std": 0.6144444067031145, "reward_change_max": 0.0, "reward_change_mean": -0.225669352337718, "reward_change_min": -0.3819498270750046, "reward_change_std": 0.1366352653130889, "reward_std": 0.6767921354621649, "rewards/accuracy_reward": 0.1250000037252903, "rewards/cosine_scaled_reward": -0.13195052836090326, "step": 23 }, { "clip_fraction": 0.0, "completion_length": 2920.9583740234375, "epoch": 0.027428571428571427, "grad_norm": 0.2333689033985138, "kl": 2.3793429136276245e-05, "lambda_div_used": 0.6070376038551331, "learning_rate": 4.6e-07, "loss": 0.1028, "reward": -0.09797604009509087, "reward_after_mean": -0.09797604009509087, "reward_after_std": 0.5090296790003777, "reward_before_mean": 0.22791288048028946, "reward_before_std": 0.49772934243083, "reward_change_max": 0.0, "reward_change_mean": -0.32588891685009, "reward_change_min": -0.5315656214952469, "reward_change_std": 0.21592886932194233, "reward_std": 0.5090296976268291, "rewards/accuracy_reward": 0.2291666716337204, "rewards/cosine_scaled_reward": -0.0012537948787212372, "step": 24 }, { "clip_fraction": 0.0, "completion_length": 2670.625045776367, "epoch": 0.02857142857142857, "grad_norm": 0.3670925199985504, "kl": 3.747781738638878e-05, "lambda_div_used": 0.6193888485431671, "learning_rate": 4.8e-07, "loss": 0.1143, "reward": -0.10888972878456116, "reward_after_mean": -0.10888972878456116, "reward_after_std": 0.5749194752424955, "reward_before_mean": 0.19831004552543163, "reward_before_std": 0.5478534679859877, "reward_change_max": 0.0, "reward_change_mean": -0.30719976499676704, "reward_change_min": -0.48458578437566757, "reward_change_std": 0.1912621408700943, "reward_std": 0.5749194994568825, "rewards/accuracy_reward": 0.2291666753590107, "rewards/cosine_scaled_reward": -0.030856636120006442, "step": 25 }, { "clip_fraction": 0.0, "completion_length": 2987.2291870117188, "epoch": 0.029714285714285714, "grad_norm": 0.17140278220176697, "kl": 2.710055559873581e-05, "lambda_div_used": 0.593448668718338, "learning_rate": 5e-07, "loss": -0.0138, "reward": -0.21064170822501183, "reward_after_mean": -0.21064170822501183, "reward_after_std": 0.46538818441331387, "reward_before_mean": 0.08803994453046471, "reward_before_std": 0.4279281683266163, "reward_change_max": 0.0, "reward_change_mean": -0.2986816558986902, "reward_change_min": -0.46159598231315613, "reward_change_std": 0.1827557273209095, "reward_std": 0.465388186275959, "rewards/accuracy_reward": 0.18750000558793545, "rewards/cosine_scaled_reward": -0.09946004673838615, "step": 26 }, { "clip_fraction": 0.0, "completion_length": 3148.0209045410156, "epoch": 0.030857142857142857, "grad_norm": 0.1689186692237854, "kl": 4.383176565170288e-05, "lambda_div_used": 0.6037615910172462, "learning_rate": 5.2e-07, "loss": 0.0438, "reward": -0.22003956139087677, "reward_after_mean": -0.22003956139087677, "reward_after_std": 0.5123853292316198, "reward_before_mean": 0.07071038894355297, "reward_before_std": 0.4823691947385669, "reward_change_max": 0.0, "reward_change_mean": -0.2907499372959137, "reward_change_min": -0.5447257719933987, "reward_change_std": 0.19816313311457634, "reward_std": 0.5123853329569101, "rewards/accuracy_reward": 0.1458333358168602, "rewards/cosine_scaled_reward": -0.07512295292690396, "step": 27 }, { "clip_fraction": 0.0, "completion_length": 2754.3542098999023, "epoch": 0.032, "grad_norm": 0.18494674563407898, "kl": 2.837367355823517e-05, "lambda_div_used": 0.6128345280885696, "learning_rate": 5.4e-07, "loss": 0.041, "reward": -0.02453425992280245, "reward_after_mean": -0.02453425992280245, "reward_after_std": 0.575501587241888, "reward_before_mean": 0.34513926785439253, "reward_before_std": 0.5150110386312008, "reward_change_max": 0.0, "reward_change_mean": -0.3696735203266144, "reward_change_min": -0.5538063198328018, "reward_change_std": 0.21833499148488045, "reward_std": 0.5755016021430492, "rewards/accuracy_reward": 0.3541666753590107, "rewards/cosine_scaled_reward": -0.009027406922541559, "step": 28 }, { "clip_fraction": 0.0, "completion_length": 3282.291717529297, "epoch": 0.03314285714285714, "grad_norm": 0.21031762659549713, "kl": 3.081187605857849e-05, "lambda_div_used": 0.5775556266307831, "learning_rate": 5.6e-07, "loss": -0.0249, "reward": -0.35000649094581604, "reward_after_mean": -0.35000649094581604, "reward_after_std": 0.4105192720890045, "reward_before_mean": -0.07036296091973782, "reward_before_std": 0.3547993768006563, "reward_change_max": 0.0, "reward_change_mean": -0.2796435412019491, "reward_change_min": -0.43543224409222603, "reward_change_std": 0.1658312752842903, "reward_std": 0.4105192758142948, "rewards/accuracy_reward": 0.08333333395421505, "rewards/cosine_scaled_reward": -0.1536962864920497, "step": 29 }, { "clip_fraction": 0.0, "completion_length": 2829.500030517578, "epoch": 0.03428571428571429, "grad_norm": 0.17382143437862396, "kl": 2.2269785404205322e-05, "lambda_div_used": 0.6104790419340134, "learning_rate": 5.8e-07, "loss": 0.0857, "reward": -0.10188953951001167, "reward_after_mean": -0.10188953951001167, "reward_after_std": 0.5173844806849957, "reward_before_mean": 0.2105449829250574, "reward_before_std": 0.516782458871603, "reward_change_max": 0.0, "reward_change_mean": -0.31243453547358513, "reward_change_min": -0.5320783108472824, "reward_change_std": 0.21172167919576168, "reward_std": 0.5173844918608665, "rewards/accuracy_reward": 0.2291666716337204, "rewards/cosine_scaled_reward": -0.018621696159243584, "step": 30 }, { "clip_fraction": 0.0, "completion_length": 3115.9791717529297, "epoch": 0.03542857142857143, "grad_norm": 0.163179412484169, "kl": 2.8448645025491714e-05, "lambda_div_used": 0.5966962277889252, "learning_rate": 6e-07, "loss": 0.0282, "reward": -0.1542623694986105, "reward_after_mean": -0.1542623694986105, "reward_after_std": 0.4793900828808546, "reward_before_mean": 0.16872040304588154, "reward_before_std": 0.44291983637958765, "reward_change_max": 0.0, "reward_change_mean": -0.3229828104376793, "reward_change_min": -0.49656323343515396, "reward_change_std": 0.19466576725244522, "reward_std": 0.47939009219408035, "rewards/accuracy_reward": 0.20833334140479565, "rewards/cosine_scaled_reward": -0.039612919092178345, "step": 31 }, { "clip_fraction": 0.0, "completion_length": 3091.312545776367, "epoch": 0.036571428571428574, "grad_norm": 0.18348075449466705, "kl": 2.8401613235473633e-05, "lambda_div_used": 0.6090468317270279, "learning_rate": 6.2e-07, "loss": 0.0689, "reward": -0.1627135332673788, "reward_after_mean": -0.1627135332673788, "reward_after_std": 0.55857895873487, "reward_before_mean": 0.13792592100799084, "reward_before_std": 0.49627261236310005, "reward_change_max": 0.0, "reward_change_mean": -0.30063946545124054, "reward_change_min": -0.4680856354534626, "reward_change_std": 0.1751015391200781, "reward_std": 0.5585789605975151, "rewards/accuracy_reward": 0.1875000074505806, "rewards/cosine_scaled_reward": -0.04957407992333174, "step": 32 }, { "clip_fraction": 0.0, "completion_length": 3420.125, "epoch": 0.037714285714285714, "grad_norm": 0.1464451402425766, "kl": 3.390759229660034e-05, "lambda_div_used": 0.6041244715452194, "learning_rate": 6.4e-07, "loss": -0.003, "reward": -0.22645239159464836, "reward_after_mean": -0.22645239159464836, "reward_after_std": 0.5185065288096666, "reward_before_mean": 0.05600981507450342, "reward_before_std": 0.48227786738425493, "reward_change_max": 0.0, "reward_change_mean": -0.2824622206389904, "reward_change_min": -0.4733004402369261, "reward_change_std": 0.18023438565433025, "reward_std": 0.5185065381228924, "rewards/accuracy_reward": 0.14583333395421505, "rewards/cosine_scaled_reward": -0.08982352539896965, "step": 33 }, { "clip_fraction": 0.0, "completion_length": 2496.0833587646484, "epoch": 0.038857142857142854, "grad_norm": 0.19927719235420227, "kl": 2.3838132619857788e-05, "lambda_div_used": 0.6616876795887947, "learning_rate": 6.6e-07, "loss": 0.0257, "reward": 0.09236510936170816, "reward_after_mean": 0.09236510936170816, "reward_after_std": 0.7538813482969999, "reward_before_mean": 0.415119782788679, "reward_before_std": 0.7498784549534321, "reward_change_max": 0.0, "reward_change_mean": -0.32275468297302723, "reward_change_min": -0.5815608035773039, "reward_change_std": 0.2209770418703556, "reward_std": 0.7538813762366772, "rewards/accuracy_reward": 0.3125000111758709, "rewards/cosine_scaled_reward": 0.10261979047209024, "step": 34 }, { "clip_fraction": 0.0, "completion_length": 2988.500030517578, "epoch": 0.04, "grad_norm": 0.21247830986976624, "kl": 3.129243850708008e-05, "lambda_div_used": 0.6234316527843475, "learning_rate": 6.800000000000001e-07, "loss": 0.0835, "reward": -0.1667571033758577, "reward_after_mean": -0.1667571033758577, "reward_after_std": 0.606516070663929, "reward_before_mean": 0.10738253593444824, "reward_before_std": 0.5717250015586615, "reward_change_max": 0.0, "reward_change_mean": -0.2741396427154541, "reward_change_min": -0.507107000797987, "reward_change_std": 0.18668675608932972, "reward_std": 0.6065160799771547, "rewards/accuracy_reward": 0.16666666977107525, "rewards/cosine_scaled_reward": -0.059284134302288294, "step": 35 }, { "clip_fraction": 0.0, "completion_length": 3297.8125, "epoch": 0.04114285714285714, "grad_norm": 0.1706894487142563, "kl": 3.0936673283576965e-05, "lambda_div_used": 0.5531303584575653, "learning_rate": 7e-07, "loss": 0.0404, "reward": -0.4647032003849745, "reward_after_mean": -0.4647032003849745, "reward_after_std": 0.31268199533224106, "reward_before_mean": -0.2008209116756916, "reward_before_std": 0.23614858835935593, "reward_change_max": 0.0, "reward_change_mean": -0.2638822831213474, "reward_change_min": -0.39729682356119156, "reward_change_std": 0.1434860322624445, "reward_std": 0.31268200278282166, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.22165424190461636, "step": 36 }, { "clip_fraction": 0.0, "completion_length": 3204.6041870117188, "epoch": 0.04228571428571429, "grad_norm": 0.16748911142349243, "kl": 2.1673738956451416e-05, "lambda_div_used": 0.5741173699498177, "learning_rate": 7.2e-07, "loss": 0.0296, "reward": -0.3909512800164521, "reward_after_mean": -0.3909512800164521, "reward_after_std": 0.41464843042194843, "reward_before_mean": -0.13006616849452257, "reward_before_std": 0.3345251912251115, "reward_change_max": 0.0, "reward_change_mean": -0.26088511012494564, "reward_change_min": -0.38747919723391533, "reward_change_std": 0.14033069927245378, "reward_std": 0.4146484360098839, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.17173283733427525, "step": 37 }, { "clip_fraction": 0.0, "completion_length": 3152.4791870117188, "epoch": 0.04342857142857143, "grad_norm": 0.14954236149787903, "kl": 2.7082860469818115e-05, "lambda_div_used": 0.5675859078764915, "learning_rate": 7.4e-07, "loss": -0.019, "reward": -0.3105775099247694, "reward_after_mean": -0.3105775099247694, "reward_after_std": 0.3737946078181267, "reward_before_mean": 0.0055494531989097595, "reward_before_std": 0.3033394878730178, "reward_change_max": 0.0, "reward_change_mean": -0.3161269761621952, "reward_change_min": -0.45622964575886726, "reward_change_std": 0.1733594536781311, "reward_std": 0.3737946189939976, "rewards/accuracy_reward": 0.12500000558793545, "rewards/cosine_scaled_reward": -0.11945055052638054, "step": 38 }, { "clip_fraction": 0.0, "completion_length": 2860.1250381469727, "epoch": 0.044571428571428574, "grad_norm": 0.1756991595029831, "kl": 1.1487558367662132e-05, "lambda_div_used": 0.5683639720082283, "learning_rate": 7.599999999999999e-07, "loss": -0.0115, "reward": -0.17498165741562843, "reward_after_mean": -0.17498165741562843, "reward_after_std": 0.417370168492198, "reward_before_mean": 0.22890623286366463, "reward_before_std": 0.3075770407449454, "reward_change_max": 0.0, "reward_change_mean": -0.40388788655400276, "reward_change_min": -0.5810245871543884, "reward_change_std": 0.22068646270781755, "reward_std": 0.41737018153071404, "rewards/accuracy_reward": 0.25000000558793545, "rewards/cosine_scaled_reward": -0.021093780174851418, "step": 39 }, { "clip_fraction": 0.0, "completion_length": 2459.9792098999023, "epoch": 0.045714285714285714, "grad_norm": 0.18066665530204773, "kl": 1.7813406884670258e-05, "lambda_div_used": 0.5652061849832535, "learning_rate": 7.799999999999999e-07, "loss": -0.05, "reward": -0.23036320134997368, "reward_after_mean": -0.23036320134997368, "reward_after_std": 0.4079273995012045, "reward_before_mean": 0.14083452709019184, "reward_before_std": 0.30146065913140774, "reward_change_max": 0.0, "reward_change_mean": -0.3711977321654558, "reward_change_min": -0.5378956347703934, "reward_change_std": 0.20801734924316406, "reward_std": 0.4079273995012045, "rewards/accuracy_reward": 0.1875, "rewards/cosine_scaled_reward": -0.04666546732187271, "step": 40 }, { "clip_fraction": 0.0, "completion_length": 3057.625045776367, "epoch": 0.046857142857142854, "grad_norm": 0.15719178318977356, "kl": 1.8415972590446472e-05, "lambda_div_used": 0.5829549804329872, "learning_rate": 8e-07, "loss": -0.0167, "reward": -0.4113044077530503, "reward_after_mean": -0.4113044077530503, "reward_after_std": 0.4361939262598753, "reward_before_mean": -0.18317487183958292, "reward_before_std": 0.37964371405541897, "reward_change_max": 0.0, "reward_change_mean": -0.2281295321881771, "reward_change_min": -0.4012086093425751, "reward_change_std": 0.1387006165459752, "reward_std": 0.43619392812252045, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.24567487463355064, "step": 41 }, { "clip_fraction": 0.0, "completion_length": 2920.208354949951, "epoch": 0.048, "grad_norm": 0.2359926551580429, "kl": 3.9443373680114746e-05, "lambda_div_used": 0.5392071008682251, "learning_rate": 8.199999999999999e-07, "loss": 0.0297, "reward": -0.5339296571910381, "reward_after_mean": -0.5339296571910381, "reward_after_std": 0.24560583755373955, "reward_before_mean": -0.2819240503013134, "reward_before_std": 0.17221506871283054, "reward_change_max": 0.0, "reward_change_mean": -0.25200559198856354, "reward_change_min": -0.3771766908466816, "reward_change_std": 0.13675262313336134, "reward_std": 0.24560584127902985, "rewards/accuracy_reward": 0.0, "rewards/cosine_scaled_reward": -0.2819240428507328, "step": 42 }, { "clip_fraction": 0.0, "completion_length": 3062.9792098999023, "epoch": 0.04914285714285714, "grad_norm": 0.17561253905296326, "kl": 2.4341046810150146e-05, "lambda_div_used": 0.5752314627170563, "learning_rate": 8.399999999999999e-07, "loss": 0.0496, "reward": -0.2924444358795881, "reward_after_mean": -0.2924444358795881, "reward_after_std": 0.3965744748711586, "reward_before_mean": 0.0118669169023633, "reward_before_std": 0.33968791645020247, "reward_change_max": 0.0, "reward_change_mean": -0.3043113648891449, "reward_change_min": -0.46210742741823196, "reward_change_std": 0.17326731327921152, "reward_std": 0.39657448418438435, "rewards/accuracy_reward": 0.12500000558793545, "rewards/cosine_scaled_reward": -0.11313308123499155, "step": 43 }, { "clip_fraction": 0.0, "completion_length": 2639.5000381469727, "epoch": 0.05028571428571429, "grad_norm": 0.28042373061180115, "kl": 7.708743214607239e-05, "lambda_div_used": 0.6106562092900276, "learning_rate": 8.599999999999999e-07, "loss": -0.0087, "reward": -0.09258423931896687, "reward_after_mean": -0.09258423931896687, "reward_after_std": 0.5288848094642162, "reward_before_mean": 0.2356700412929058, "reward_before_std": 0.5172735182568431, "reward_change_max": 0.0, "reward_change_mean": -0.3282542694360018, "reward_change_min": -0.5651697888970375, "reward_change_std": 0.21937444992363453, "reward_std": 0.5288848392665386, "rewards/accuracy_reward": 0.1875000037252903, "rewards/cosine_scaled_reward": 0.04817003011703491, "step": 44 }, { "clip_fraction": 0.0, "completion_length": 3355.5833740234375, "epoch": 0.05142857142857143, "grad_norm": 0.1452389359474182, "kl": 3.167241811752319e-05, "lambda_div_used": 0.6173592880368233, "learning_rate": 8.799999999999999e-07, "loss": -0.0109, "reward": -0.19022756442427635, "reward_after_mean": -0.19022756442427635, "reward_after_std": 0.5816491153091192, "reward_before_mean": 0.07632183004170656, "reward_before_std": 0.5414999173954129, "reward_change_max": 0.0, "reward_change_mean": -0.26654940098524094, "reward_change_min": -0.4476119540631771, "reward_change_std": 0.16907985042780638, "reward_std": 0.5816491302102804, "rewards/accuracy_reward": 0.1666666716337204, "rewards/cosine_scaled_reward": -0.09034484624862671, "step": 45 }, { "clip_fraction": 0.0, "completion_length": 3222.1250534057617, "epoch": 0.052571428571428575, "grad_norm": 0.2406596839427948, "kl": 3.855302929878235e-05, "lambda_div_used": 0.5541960597038269, "learning_rate": 9e-07, "loss": 0.0131, "reward": -0.502369549125433, "reward_after_mean": -0.502369549125433, "reward_after_std": 0.31580126471817493, "reward_before_mean": -0.2632645908743143, "reward_before_std": 0.24160117655992508, "reward_change_max": 0.0, "reward_change_mean": -0.2391049787402153, "reward_change_min": -0.3644678257405758, "reward_change_std": 0.13095823675394058, "reward_std": 0.31580127589404583, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.2840979266911745, "step": 46 }, { "clip_fraction": 0.0, "completion_length": 2855.7083587646484, "epoch": 0.053714285714285714, "grad_norm": 0.221223384141922, "kl": 5.85503876209259e-05, "lambda_div_used": 0.6433944702148438, "learning_rate": 9.2e-07, "loss": -0.035, "reward": -0.005535580217838287, "reward_after_mean": -0.005535580217838287, "reward_after_std": 0.6469592582434416, "reward_before_mean": 0.29989274591207504, "reward_before_std": 0.6766865756362677, "reward_change_max": 0.0, "reward_change_mean": -0.3054283373057842, "reward_change_min": -0.5757316760718822, "reward_change_std": 0.23097991570830345, "reward_std": 0.6469592582434416, "rewards/accuracy_reward": 0.2708333358168602, "rewards/cosine_scaled_reward": 0.029059413820505142, "step": 47 }, { "clip_fraction": 0.0, "completion_length": 2878.000015258789, "epoch": 0.054857142857142854, "grad_norm": 0.2013225257396698, "kl": 5.435943603515625e-05, "lambda_div_used": 0.5998551249504089, "learning_rate": 9.399999999999999e-07, "loss": -0.003, "reward": -0.2259172461926937, "reward_after_mean": -0.2259172461926937, "reward_after_std": 0.49600384198129177, "reward_before_mean": 0.07100770622491837, "reward_before_std": 0.4581450503319502, "reward_change_max": 0.0, "reward_change_mean": -0.2969249580055475, "reward_change_min": -0.49217061325907707, "reward_change_std": 0.1858653174713254, "reward_std": 0.4960038587450981, "rewards/accuracy_reward": 0.16666667349636555, "rewards/cosine_scaled_reward": -0.09565895947162062, "step": 48 }, { "clip_fraction": 0.0, "completion_length": 2392.937530517578, "epoch": 0.056, "grad_norm": 0.2075609713792801, "kl": 2.6391353458166122e-05, "lambda_div_used": 0.5920819342136383, "learning_rate": 9.6e-07, "loss": -0.0022, "reward": -0.19488872308284044, "reward_after_mean": -0.19488872308284044, "reward_after_std": 0.5334667162969708, "reward_before_mean": 0.13957323785871267, "reward_before_std": 0.42348560388199985, "reward_change_max": 0.0, "reward_change_mean": -0.3344619367271662, "reward_change_min": -0.4920482262969017, "reward_change_std": 0.18814518116414547, "reward_std": 0.5334667414426804, "rewards/accuracy_reward": 0.2291666679084301, "rewards/cosine_scaled_reward": -0.08959344774484634, "step": 49 }, { "clip_fraction": 0.0, "completion_length": 2806.43754196167, "epoch": 0.05714285714285714, "grad_norm": 0.19358737766742706, "kl": 4.048272967338562e-05, "lambda_div_used": 0.6027668565511703, "learning_rate": 9.8e-07, "loss": 0.0105, "reward": 0.012562461197376251, "reward_after_mean": 0.012562461197376251, "reward_after_std": 0.5324771385639906, "reward_before_mean": 0.43131681717932224, "reward_before_std": 0.4746437296271324, "reward_change_max": 0.0, "reward_change_mean": -0.4187543373554945, "reward_change_min": -0.656049095094204, "reward_change_std": 0.25779934599995613, "reward_std": 0.5324771534651518, "rewards/accuracy_reward": 0.3541666753590107, "rewards/cosine_scaled_reward": 0.0771501250565052, "step": 50 }, { "clip_fraction": 0.0, "completion_length": 2426.833366394043, "epoch": 0.05828571428571429, "grad_norm": 0.20888112485408783, "kl": 0.00012753717601299286, "lambda_div_used": 0.5862141028046608, "learning_rate": 1e-06, "loss": 0.0473, "reward": -0.2718443821067922, "reward_after_mean": -0.2718443821067922, "reward_after_std": 0.4596429727971554, "reward_before_mean": 0.013651263900101185, "reward_before_std": 0.3922702055424452, "reward_change_max": 0.0, "reward_change_mean": -0.28549565002322197, "reward_change_min": -0.4343360997736454, "reward_change_std": 0.16514169331640005, "reward_std": 0.4596429765224457, "rewards/accuracy_reward": 0.1250000037252903, "rewards/cosine_scaled_reward": -0.11134873703122139, "step": 51 }, { "clip_fraction": 0.0, "completion_length": 2778.0208854675293, "epoch": 0.05942857142857143, "grad_norm": 0.21131280064582825, "kl": 7.995869964361191e-05, "lambda_div_used": 0.6727548465132713, "learning_rate": 9.999890338174275e-07, "loss": 0.0069, "reward": 0.09182125888764858, "reward_after_mean": 0.09182125888764858, "reward_after_std": 0.7875823080539703, "reward_before_mean": 0.39299324713647366, "reward_before_std": 0.8082237709313631, "reward_change_max": 0.0, "reward_change_mean": -0.3011719882488251, "reward_change_min": -0.5991150513291359, "reward_change_std": 0.23383105359971523, "reward_std": 0.7875823173671961, "rewards/accuracy_reward": 0.31250000186264515, "rewards/cosine_scaled_reward": 0.08049324085004628, "step": 52 }, { "clip_fraction": 0.0, "completion_length": 2853.5000381469727, "epoch": 0.060571428571428575, "grad_norm": 0.22672921419143677, "kl": 6.973370909690857e-05, "lambda_div_used": 0.6237443387508392, "learning_rate": 9.999561358041868e-07, "loss": 0.0362, "reward": -0.1672458229586482, "reward_after_mean": -0.1672458229586482, "reward_after_std": 0.5996230188757181, "reward_before_mean": 0.10632886923849583, "reward_before_std": 0.5751665635034442, "reward_change_max": 0.0, "reward_change_mean": -0.27357468008995056, "reward_change_min": -0.5130747817456722, "reward_change_std": 0.18984936457127333, "reward_std": 0.599623030051589, "rewards/accuracy_reward": 0.18750000558793545, "rewards/cosine_scaled_reward": -0.08117113448679447, "step": 53 }, { "clip_fraction": 0.0, "completion_length": 2885.5625610351562, "epoch": 0.061714285714285715, "grad_norm": 0.1966475397348404, "kl": 5.410425364971161e-05, "lambda_div_used": 0.6474145576357841, "learning_rate": 9.999013075636804e-07, "loss": 0.0914, "reward": 0.08282669354230165, "reward_after_mean": 0.08282669354230165, "reward_after_std": 0.6879531536251307, "reward_before_mean": 0.4185080798342824, "reward_before_std": 0.6889733774587512, "reward_change_max": 0.0, "reward_change_mean": -0.3356813807040453, "reward_change_min": -0.5831126980483532, "reward_change_std": 0.23159859981387854, "reward_std": 0.6879531759768724, "rewards/accuracy_reward": 0.31250000931322575, "rewards/cosine_scaled_reward": 0.10600805375725031, "step": 54 }, { "clip_fraction": 0.0, "completion_length": 3030.291679382324, "epoch": 0.06285714285714286, "grad_norm": 0.1828983873128891, "kl": 7.626600563526154e-05, "lambda_div_used": 0.6072710752487183, "learning_rate": 9.998245517681593e-07, "loss": 0.0472, "reward": -0.1825969135388732, "reward_after_mean": -0.1825969135388732, "reward_after_std": 0.5332597587257624, "reward_before_mean": 0.1025706585496664, "reward_before_std": 0.5008249981328845, "reward_change_max": 0.0, "reward_change_mean": -0.2851675618439913, "reward_change_min": -0.46727684512734413, "reward_change_std": 0.1838911110535264, "reward_std": 0.5332597624510527, "rewards/accuracy_reward": 0.16666666977107525, "rewards/cosine_scaled_reward": -0.06409600656479597, "step": 55 }, { "clip_fraction": 0.0, "completion_length": 2967.479202270508, "epoch": 0.064, "grad_norm": 0.17583292722702026, "kl": 2.273544669151306e-05, "lambda_div_used": 0.5603753849864006, "learning_rate": 9.997258721585931e-07, "loss": 0.0323, "reward": -0.1957027204334736, "reward_after_mean": -0.1957027204334736, "reward_after_std": 0.38146832399070263, "reward_before_mean": 0.20488426089286804, "reward_before_std": 0.2763750562444329, "reward_change_max": 0.0, "reward_change_mean": -0.4005869887769222, "reward_change_min": -0.5782447755336761, "reward_change_std": 0.2227842453867197, "reward_std": 0.3814683295786381, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.0034490730613470078, "step": 56 }, { "clip_fraction": 0.0, "completion_length": 3231.250045776367, "epoch": 0.06514285714285714, "grad_norm": 0.12549880146980286, "kl": 1.6013160347938538e-05, "lambda_div_used": 0.6204828321933746, "learning_rate": 9.996052735444862e-07, "loss": -0.0046, "reward": -0.2907296810299158, "reward_after_mean": -0.2907296810299158, "reward_after_std": 0.6158930230885744, "reward_before_mean": -0.06423777744930703, "reward_before_std": 0.5571717582643032, "reward_change_max": 0.0, "reward_change_mean": -0.22649191319942474, "reward_change_min": -0.3973502218723297, "reward_change_std": 0.13921643886715174, "reward_std": 0.6158930379897356, "rewards/accuracy_reward": 0.10416666977107525, "rewards/cosine_scaled_reward": -0.16840444970875978, "step": 57 }, { "clip_fraction": 0.0, "completion_length": 2334.104217529297, "epoch": 0.06628571428571428, "grad_norm": 0.2545982599258423, "kl": 0.0004110261797904968, "lambda_div_used": 0.6491317972540855, "learning_rate": 9.994627618036452e-07, "loss": 0.0807, "reward": 0.020275337621569633, "reward_after_mean": 0.020275337621569633, "reward_after_std": 0.6884118355810642, "reward_before_mean": 0.33974294923245907, "reward_before_std": 0.6970602702349424, "reward_change_max": 0.0, "reward_change_mean": -0.31946760788559914, "reward_change_min": -0.6263015754520893, "reward_change_std": 0.23682032711803913, "reward_std": 0.6884118635207415, "rewards/accuracy_reward": 0.2916666753590107, "rewards/cosine_scaled_reward": 0.0480762617662549, "step": 58 }, { "clip_fraction": 0.0, "completion_length": 2802.7916946411133, "epoch": 0.06742857142857143, "grad_norm": 0.15832675993442535, "kl": 1.6205012798309326e-05, "lambda_div_used": 0.5855341628193855, "learning_rate": 9.992983438818915e-07, "loss": 0.0556, "reward": -0.16703030467033386, "reward_after_mean": -0.16703030467033386, "reward_after_std": 0.43107361160218716, "reward_before_mean": 0.1723417080938816, "reward_before_std": 0.3897872269153595, "reward_change_max": 0.0, "reward_change_mean": -0.33937204256653786, "reward_change_min": -0.5048820599913597, "reward_change_std": 0.20309140533208847, "reward_std": 0.43107362277805805, "rewards/accuracy_reward": 0.20833334140479565, "rewards/cosine_scaled_reward": -0.03599160863086581, "step": 59 }, { "clip_fraction": 0.0, "completion_length": 2924.7708892822266, "epoch": 0.06857142857142857, "grad_norm": 0.1720697283744812, "kl": 4.5239925384521484e-05, "lambda_div_used": 0.593461163341999, "learning_rate": 9.991120277927223e-07, "loss": 0.0057, "reward": -0.3369075497612357, "reward_after_mean": -0.3369075497612357, "reward_after_std": 0.48820602521300316, "reward_before_mean": -0.0864023957401514, "reward_before_std": 0.4260971210896969, "reward_change_max": 0.0, "reward_change_mean": -0.25050514191389084, "reward_change_min": -0.42594166472554207, "reward_change_std": 0.14879687502980232, "reward_std": 0.4882060382515192, "rewards/accuracy_reward": 0.10416666977107525, "rewards/cosine_scaled_reward": -0.19056908111087978, "step": 60 }, { "clip_fraction": 0.0, "completion_length": 2930.2083740234375, "epoch": 0.06971428571428571, "grad_norm": 0.14971715211868286, "kl": 0.00015359371900558472, "lambda_div_used": 0.5796276479959488, "learning_rate": 9.989038226169207e-07, "loss": 0.0439, "reward": -0.19501841347664595, "reward_after_mean": -0.19501841347664595, "reward_after_std": 0.48850303143262863, "reward_before_mean": 0.1854183403775096, "reward_before_std": 0.361351037863642, "reward_change_max": 0.0, "reward_change_mean": -0.3804367668926716, "reward_change_min": -0.5838541947305202, "reward_change_std": 0.21505682356655598, "reward_std": 0.4885030463337898, "rewards/accuracy_reward": 0.22916666977107525, "rewards/cosine_scaled_reward": -0.04374831053428352, "step": 61 }, { "clip_fraction": 0.0, "completion_length": 2613.8750610351562, "epoch": 0.07085714285714285, "grad_norm": 0.2425049990415573, "kl": 0.0003200247883796692, "lambda_div_used": 0.624903179705143, "learning_rate": 9.98673738502114e-07, "loss": 0.0167, "reward": -0.08396586682647467, "reward_after_mean": -0.08396586682647467, "reward_after_std": 0.6689736172556877, "reward_before_mean": 0.24665060732513666, "reward_before_std": 0.5764260310679674, "reward_change_max": 0.0, "reward_change_mean": -0.33061649464070797, "reward_change_min": -0.5171178039163351, "reward_change_std": 0.1906643807888031, "reward_std": 0.6689736507833004, "rewards/accuracy_reward": 0.25000000186264515, "rewards/cosine_scaled_reward": -0.003349396400153637, "step": 62 }, { "clip_fraction": 0.0, "completion_length": 2381.5834045410156, "epoch": 0.072, "grad_norm": 0.19860316812992096, "kl": 0.0002915412187576294, "lambda_div_used": 0.6054271757602692, "learning_rate": 9.98421786662277e-07, "loss": -0.0534, "reward": -0.08262127637863159, "reward_after_mean": -0.08262127637863159, "reward_after_std": 0.5753348544239998, "reward_before_mean": 0.2834562277421355, "reward_before_std": 0.49130255077034235, "reward_change_max": 0.0, "reward_change_mean": -0.36607749573886395, "reward_change_min": -0.5886767171323299, "reward_change_std": 0.22193934302777052, "reward_std": 0.5753348786383867, "rewards/accuracy_reward": 0.25000000186264515, "rewards/cosine_scaled_reward": 0.03345622168853879, "step": 63 }, { "clip_fraction": 0.0, "completion_length": 2768.2291946411133, "epoch": 0.07314285714285715, "grad_norm": 0.2018764764070511, "kl": 0.00010337494313716888, "lambda_div_used": 0.5978562384843826, "learning_rate": 9.981479793771866e-07, "loss": 0.0675, "reward": -0.15635946393013, "reward_after_mean": -0.15635946393013, "reward_after_std": 0.4916374906897545, "reward_before_mean": 0.16407555295154452, "reward_before_std": 0.4476189352571964, "reward_change_max": 0.0, "reward_change_mean": -0.32043501175940037, "reward_change_min": -0.4865843001753092, "reward_change_std": 0.19141005631536245, "reward_std": 0.4916375018656254, "rewards/accuracy_reward": 0.20833334140479565, "rewards/cosine_scaled_reward": -0.04425778239965439, "step": 64 }, { "clip_fraction": 0.0, "completion_length": 2693.0833702087402, "epoch": 0.07428571428571429, "grad_norm": 0.22227466106414795, "kl": 0.000233454629778862, "lambda_div_used": 0.5729491114616394, "learning_rate": 9.97852329991824e-07, "loss": -0.0029, "reward": -0.29786941036581993, "reward_after_mean": -0.29786941036581993, "reward_after_std": 0.4606590736657381, "reward_before_mean": 0.024189180694520473, "reward_before_std": 0.3302721520885825, "reward_change_max": 0.0, "reward_change_mean": -0.32205857522785664, "reward_change_min": -0.4435257241129875, "reward_change_std": 0.16619026195257902, "reward_std": 0.46065907552838326, "rewards/accuracy_reward": 0.1666666679084301, "rewards/cosine_scaled_reward": -0.14247749373316765, "step": 65 }, { "clip_fraction": 0.0, "completion_length": 2150.6250076293945, "epoch": 0.07542857142857143, "grad_norm": 0.24965105950832367, "kl": 0.00029963254928588867, "lambda_div_used": 0.5969655886292458, "learning_rate": 9.975348529157229e-07, "loss": -0.0164, "reward": -0.09659457858651876, "reward_after_mean": -0.09659457858651876, "reward_after_std": 0.5355625227093697, "reward_before_mean": 0.2962189055979252, "reward_before_std": 0.4444810135755688, "reward_change_max": 0.0, "reward_change_mean": -0.3928134962916374, "reward_change_min": -0.6039444245398045, "reward_change_std": 0.23178349621593952, "reward_std": 0.5355625320225954, "rewards/accuracy_reward": 0.29166667349636555, "rewards/cosine_scaled_reward": 0.004552240949124098, "step": 66 }, { "clip_fraction": 0.0, "completion_length": 3400.687530517578, "epoch": 0.07657142857142857, "grad_norm": 0.15669777989387512, "kl": 0.00033114850521087646, "lambda_div_used": 0.584111362695694, "learning_rate": 9.971955636222684e-07, "loss": -0.0191, "reward": -0.4098571501672268, "reward_after_mean": -0.4098571501672268, "reward_after_std": 0.4613211713731289, "reward_before_mean": -0.18392361979931593, "reward_before_std": 0.3809601026587188, "reward_change_max": 0.0, "reward_change_mean": -0.22593354433774948, "reward_change_min": -0.33898190036416054, "reward_change_std": 0.12169961258769035, "reward_std": 0.4613211937248707, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.24642362166196108, "step": 67 }, { "clip_fraction": 0.0, "completion_length": 2278.854232788086, "epoch": 0.07771428571428571, "grad_norm": 0.42183881998062134, "kl": 0.001455545425415039, "lambda_div_used": 0.6024054288864136, "learning_rate": 9.968344786479415e-07, "loss": 0.2004, "reward": -0.20893910806626081, "reward_after_mean": -0.20893910806626081, "reward_after_std": 0.501633994281292, "reward_before_mean": 0.08473129197955132, "reward_before_std": 0.4724907008931041, "reward_change_max": 0.0, "reward_change_mean": -0.2936703860759735, "reward_change_min": -0.5323524177074432, "reward_change_std": 0.1964686680585146, "reward_std": 0.501633994281292, "rewards/accuracy_reward": 0.1666666716337204, "rewards/cosine_scaled_reward": -0.08193538710474968, "step": 68 }, { "clip_fraction": 0.0, "completion_length": 2686.9792404174805, "epoch": 0.07885714285714286, "grad_norm": 0.21581123769283295, "kl": 0.001392066478729248, "lambda_div_used": 0.603032223880291, "learning_rate": 9.964516155915151e-07, "loss": 0.0433, "reward": -0.30412171594798565, "reward_after_mean": -0.30412171594798565, "reward_after_std": 0.5522229336202145, "reward_before_mean": -0.06000571511685848, "reward_before_std": 0.46599213033914566, "reward_change_max": 0.0, "reward_change_mean": -0.24411599524319172, "reward_change_min": -0.35635631531476974, "reward_change_std": 0.13294454757124186, "reward_std": 0.5522229373455048, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.14333904418163002, "step": 69 }, { "clip_fraction": 0.0, "completion_length": 3073.083366394043, "epoch": 0.08, "grad_norm": 0.2320345938205719, "kl": 0.0010668635368347168, "lambda_div_used": 0.5898077040910721, "learning_rate": 9.960469931131936e-07, "loss": 0.0121, "reward": -0.35871684923768044, "reward_after_mean": -0.35871684923768044, "reward_after_std": 0.4651712905615568, "reward_before_mean": -0.11179051315411925, "reward_before_std": 0.4065481722354889, "reward_change_max": 0.0, "reward_change_mean": -0.24692632257938385, "reward_change_min": -0.4165010005235672, "reward_change_std": 0.14658983051776886, "reward_std": 0.46517129614949226, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.19512385316193104, "step": 70 }, { "clip_fraction": 0.0, "completion_length": 2744.0833740234375, "epoch": 0.08114285714285714, "grad_norm": 0.5214744210243225, "kl": 0.0015140660107135773, "lambda_div_used": 0.5972233712673187, "learning_rate": 9.956206309337066e-07, "loss": -0.0488, "reward": -0.16986532509326935, "reward_after_mean": -0.16986532509326935, "reward_after_std": 0.4745449274778366, "reward_before_mean": 0.14809664152562618, "reward_before_std": 0.4448474030941725, "reward_change_max": 0.0, "reward_change_mean": -0.3179619759321213, "reward_change_min": -0.5534767471253872, "reward_change_std": 0.20240054093301296, "reward_std": 0.4745449423789978, "rewards/accuracy_reward": 0.1875000074505806, "rewards/cosine_scaled_reward": -0.03940335847437382, "step": 71 }, { "clip_fraction": 0.0, "completion_length": 3218.729217529297, "epoch": 0.08228571428571428, "grad_norm": 0.16154354810714722, "kl": 0.0012544244527816772, "lambda_div_used": 0.557806558907032, "learning_rate": 9.951725498333448e-07, "loss": 0.0064, "reward": -0.47252254374325275, "reward_after_mean": -0.47252254374325275, "reward_after_std": 0.31631129793822765, "reward_before_mean": -0.21749469326459803, "reward_before_std": 0.25799814565107226, "reward_change_max": 0.0, "reward_change_mean": -0.2550278529524803, "reward_change_min": -0.39039382711052895, "reward_change_std": 0.1440178118646145, "reward_std": 0.3163113035261631, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.23832802660763264, "step": 72 }, { "clip_fraction": 0.0, "completion_length": 3557.8958740234375, "epoch": 0.08342857142857144, "grad_norm": 0.14565619826316833, "kl": 0.00024437904357910156, "lambda_div_used": 0.6077284440398216, "learning_rate": 9.947027716509488e-07, "loss": 0.0022, "reward": -0.2573185842484236, "reward_after_mean": -0.2573185842484236, "reward_after_std": 0.5269294902682304, "reward_before_mean": 0.003301744582131505, "reward_before_std": 0.49822407588362694, "reward_change_max": 0.0, "reward_change_mean": -0.2606203146278858, "reward_change_min": -0.4737945795059204, "reward_change_std": 0.1746126189827919, "reward_std": 0.5269295033067465, "rewards/accuracy_reward": 0.12500000186264515, "rewards/cosine_scaled_reward": -0.12169826403260231, "step": 73 }, { "clip_fraction": 0.0, "completion_length": 3327.5416870117188, "epoch": 0.08457142857142858, "grad_norm": 0.14613939821720123, "kl": 0.0008155256509780884, "lambda_div_used": 0.5811568349599838, "learning_rate": 9.942113192828444e-07, "loss": 0.0434, "reward": -0.18079723790287971, "reward_after_mean": -0.18079723790287971, "reward_after_std": 0.47902655228972435, "reward_before_mean": 0.17984693683683872, "reward_before_std": 0.37502646166831255, "reward_change_max": 0.0, "reward_change_mean": -0.3606441728770733, "reward_change_min": -0.5273178145289421, "reward_change_std": 0.2027215762063861, "reward_std": 0.4790265541523695, "rewards/accuracy_reward": 0.20833333395421505, "rewards/cosine_scaled_reward": -0.02848641388118267, "step": 74 }, { "clip_fraction": 0.0, "completion_length": 3048.625045776367, "epoch": 0.08571428571428572, "grad_norm": 0.15286406874656677, "kl": 0.0011966601014137268, "lambda_div_used": 0.5748990103602409, "learning_rate": 9.93698216681727e-07, "loss": 0.0437, "reward": -0.2083893045783043, "reward_after_mean": -0.2083893045783043, "reward_after_std": 0.460305891931057, "reward_before_mean": 0.18028387241065502, "reward_before_std": 0.34195839799940586, "reward_change_max": 0.0, "reward_change_mean": -0.3886731844395399, "reward_change_min": -0.5808934159576893, "reward_change_std": 0.21857514511793852, "reward_std": 0.46030591055750847, "rewards/accuracy_reward": 0.20833333395421505, "rewards/cosine_scaled_reward": -0.02804945968091488, "step": 75 }, { "clip_fraction": 0.0, "completion_length": 2946.9583740234375, "epoch": 0.08685714285714285, "grad_norm": 0.1700589507818222, "kl": 0.0001853257417678833, "lambda_div_used": 0.5423546582460403, "learning_rate": 9.931634888554935e-07, "loss": -0.0337, "reward": -0.5232770070433617, "reward_after_mean": -0.5232770070433617, "reward_after_std": 0.25481574051082134, "reward_before_mean": -0.26925108954310417, "reward_before_std": 0.18604754656553268, "reward_change_max": 0.0, "reward_change_mean": -0.2540259212255478, "reward_change_min": -0.395156804472208, "reward_change_std": 0.1399830151349306, "reward_std": 0.25481574423611164, "rewards/accuracy_reward": 0.0, "rewards/cosine_scaled_reward": -0.26925108954310417, "step": 76 }, { "clip_fraction": 0.0, "completion_length": 3240.5416870117188, "epoch": 0.088, "grad_norm": 0.1512913703918457, "kl": 0.0002654418349266052, "lambda_div_used": 0.5648113340139389, "learning_rate": 9.926071618660237e-07, "loss": 0.0088, "reward": -0.3492433689534664, "reward_after_mean": -0.3492433689534664, "reward_after_std": 0.342065442353487, "reward_before_mean": -0.056707512587308884, "reward_before_std": 0.2952875215560198, "reward_change_max": 0.0, "reward_change_mean": -0.29253583773970604, "reward_change_min": -0.45687081664800644, "reward_change_std": 0.17306052893400192, "reward_std": 0.3420654535293579, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.14004085958003998, "step": 77 }, { "clip_fraction": 0.0, "completion_length": 3275.8333740234375, "epoch": 0.08914285714285715, "grad_norm": 0.15637144446372986, "kl": 0.0002499721013009548, "lambda_div_used": 0.5740868151187897, "learning_rate": 9.9202926282791e-07, "loss": 0.0114, "reward": -0.19711515866219997, "reward_after_mean": -0.19711515866219997, "reward_after_std": 0.45171595364809036, "reward_before_mean": 0.16380855813622475, "reward_before_std": 0.3395942933857441, "reward_change_max": 0.0, "reward_change_mean": -0.36092369817197323, "reward_change_min": -0.5169509537518024, "reward_change_std": 0.20019148755818605, "reward_std": 0.45171597972512245, "rewards/accuracy_reward": 0.22916666977107525, "rewards/cosine_scaled_reward": -0.06535813398659229, "step": 78 }, { "clip_fraction": 0.0, "completion_length": 2265.5833702087402, "epoch": 0.09028571428571429, "grad_norm": 0.2618561089038849, "kl": 0.0011730790138244629, "lambda_div_used": 0.600786842405796, "learning_rate": 9.91429819907136e-07, "loss": -0.0868, "reward": -0.17444273456931114, "reward_after_mean": -0.17444273456931114, "reward_after_std": 0.48976252786815166, "reward_before_mean": 0.14442736096680164, "reward_before_std": 0.46738001704216003, "reward_change_max": 0.0, "reward_change_mean": -0.3188701141625643, "reward_change_min": -0.5393733121454716, "reward_change_std": 0.2091370914131403, "reward_std": 0.4897625334560871, "rewards/accuracy_reward": 0.2083333395421505, "rewards/cosine_scaled_reward": -0.0639059729874134, "step": 79 }, { "clip_fraction": 0.0, "completion_length": 3362.7291870117188, "epoch": 0.09142857142857143, "grad_norm": 0.1623566746711731, "kl": 0.0009048879146575928, "lambda_div_used": 0.5854988992214203, "learning_rate": 9.908088623197048e-07, "loss": 0.0085, "reward": -0.30083255702629685, "reward_after_mean": -0.30083255702629685, "reward_after_std": 0.4121339116245508, "reward_before_mean": -0.016935506835579872, "reward_before_std": 0.3900100700557232, "reward_change_max": 0.0, "reward_change_mean": -0.2838970310986042, "reward_change_min": -0.5064741894602776, "reward_change_std": 0.18459498137235641, "reward_std": 0.41213392093777657, "rewards/accuracy_reward": 0.1250000037252903, "rewards/cosine_scaled_reward": -0.1419355261605233, "step": 80 }, { "clip_fraction": 0.0, "completion_length": 3174.729179382324, "epoch": 0.09257142857142857, "grad_norm": 0.32535243034362793, "kl": 0.0019363164901733398, "lambda_div_used": 0.5821651369333267, "learning_rate": 9.901664203302124e-07, "loss": -0.0374, "reward": -0.32319752499461174, "reward_after_mean": -0.32319752499461174, "reward_after_std": 0.4326016325503588, "reward_before_mean": -0.05671766586601734, "reward_before_std": 0.3768819263204932, "reward_change_max": 0.0, "reward_change_mean": -0.26647986844182014, "reward_change_min": -0.4188438169658184, "reward_change_std": 0.15899487026035786, "reward_std": 0.4326016325503588, "rewards/accuracy_reward": 0.10416666977107525, "rewards/cosine_scaled_reward": -0.16088432881224435, "step": 81 }, { "clip_fraction": 0.0, "completion_length": 2768.7708587646484, "epoch": 0.09371428571428571, "grad_norm": 0.1785093992948532, "kl": 0.0016040951013565063, "lambda_div_used": 0.6084312722086906, "learning_rate": 9.895025252503755e-07, "loss": -0.0708, "reward": -0.04713406786322594, "reward_after_mean": -0.04713406786322594, "reward_after_std": 0.5625460837036371, "reward_before_mean": 0.3383842948824167, "reward_before_std": 0.501438532024622, "reward_change_max": 0.0, "reward_change_mean": -0.3855183683335781, "reward_change_min": -0.6507172286510468, "reward_change_std": 0.24704305455088615, "reward_std": 0.5625460930168629, "rewards/accuracy_reward": 0.2916666679084301, "rewards/cosine_scaled_reward": 0.0467176353558898, "step": 82 }, { "clip_fraction": 0.0, "completion_length": 2859.7083587646484, "epoch": 0.09485714285714286, "grad_norm": 0.19722098112106323, "kl": 0.0007025524973869324, "lambda_div_used": 0.5633128806948662, "learning_rate": 9.888172094375033e-07, "loss": -0.0067, "reward": -0.39044155552983284, "reward_after_mean": -0.39044155552983284, "reward_after_std": 0.326798003166914, "reward_before_mean": -0.11264067143201828, "reward_before_std": 0.28597197867929935, "reward_change_max": 0.0, "reward_change_mean": -0.27780089154839516, "reward_change_min": -0.4442596957087517, "reward_change_std": 0.16762787476181984, "reward_std": 0.32679800875484943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.19597399793565273, "step": 83 }, { "clip_fraction": 0.0, "completion_length": 3121.0000610351562, "epoch": 0.096, "grad_norm": 0.16867555677890778, "kl": 0.0010808110237121582, "lambda_div_used": 0.6293942183256149, "learning_rate": 9.881105062929221e-07, "loss": 0.0125, "reward": -0.10214579105377197, "reward_after_mean": -0.10214579105377197, "reward_after_std": 0.6248092297464609, "reward_before_mean": 0.18623400200158358, "reward_before_std": 0.6097953664138913, "reward_change_max": 0.0, "reward_change_mean": -0.2883797585964203, "reward_change_min": -0.51039794459939, "reward_change_std": 0.20119250286370516, "reward_std": 0.6248092465102673, "rewards/accuracy_reward": 0.20833333767950535, "rewards/cosine_scaled_reward": -0.022099352441728115, "step": 84 }, { "clip_fraction": 0.0, "completion_length": 3096.4375610351562, "epoch": 0.09714285714285714, "grad_norm": 0.15777070820331573, "kl": 0.00037366151809692383, "lambda_div_used": 0.5622768849134445, "learning_rate": 9.873824502603459e-07, "loss": -0.0085, "reward": -0.42790209501981735, "reward_after_mean": -0.42790209501981735, "reward_after_std": 0.33375417813658714, "reward_before_mean": -0.1516575189307332, "reward_before_std": 0.28144488483667374, "reward_change_max": 0.0, "reward_change_mean": -0.2762445732951164, "reward_change_min": -0.4674489162862301, "reward_change_std": 0.1669005323201418, "reward_std": 0.33375419303774834, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.19332418963313103, "step": 85 }, { "clip_fraction": 0.0, "completion_length": 3210.5833740234375, "epoch": 0.09828571428571428, "grad_norm": 0.1625698059797287, "kl": 0.0013466477394104004, "lambda_div_used": 0.5538754910230637, "learning_rate": 9.866330768241983e-07, "loss": 0.0488, "reward": -0.33031318336725235, "reward_after_mean": -0.33031318336725235, "reward_after_std": 0.37777717411518097, "reward_before_mean": 0.018868350191041827, "reward_before_std": 0.24134167935699224, "reward_change_max": 0.0, "reward_change_mean": -0.349181542173028, "reward_change_min": -0.48746827244758606, "reward_change_std": 0.18057182617485523, "reward_std": 0.3777771834284067, "rewards/accuracy_reward": 0.14583333395421505, "rewards/cosine_scaled_reward": -0.1269649676978588, "step": 86 }, { "clip_fraction": 0.0, "completion_length": 3062.791679382324, "epoch": 0.09942857142857142, "grad_norm": 0.18156039714813232, "kl": 0.0014489144086837769, "lambda_div_used": 0.5805760622024536, "learning_rate": 9.85862422507884e-07, "loss": 0.0535, "reward": -0.3212639403063804, "reward_after_mean": -0.3212639403063804, "reward_after_std": 0.42872031405568123, "reward_before_mean": -0.04342861473560333, "reward_before_std": 0.36875712499022484, "reward_change_max": 0.0, "reward_change_mean": -0.277835326269269, "reward_change_min": -0.4430685229599476, "reward_change_std": 0.16383375227451324, "reward_std": 0.4287203270941973, "rewards/accuracy_reward": 0.08333333395421505, "rewards/cosine_scaled_reward": -0.12676194217056036, "step": 87 }, { "clip_fraction": 0.0, "completion_length": 2671.5000610351562, "epoch": 0.10057142857142858, "grad_norm": 0.20773273706436157, "kl": 0.0018809139728546143, "lambda_div_used": 0.6715577393770218, "learning_rate": 9.850705248720068e-07, "loss": 0.0865, "reward": 0.05163753964006901, "reward_after_mean": 0.05163753964006901, "reward_after_std": 0.8082198165357113, "reward_before_mean": 0.33743299047637265, "reward_before_std": 0.8022481258958578, "reward_change_max": 0.0, "reward_change_mean": -0.28579549118876457, "reward_change_min": -0.5569394677877426, "reward_change_std": 0.20709870103746653, "reward_std": 0.8082198724150658, "rewards/accuracy_reward": 0.2916666753590107, "rewards/cosine_scaled_reward": 0.0457663563429378, "step": 88 }, { "clip_fraction": 0.0, "completion_length": 3201.916717529297, "epoch": 0.10171428571428572, "grad_norm": 0.17959482967853546, "kl": 0.002676248550415039, "lambda_div_used": 0.610780768096447, "learning_rate": 9.8425742251254e-07, "loss": 0.0729, "reward": -0.15698900260031223, "reward_after_mean": -0.15698900260031223, "reward_after_std": 0.560118380934, "reward_before_mean": 0.1408877931535244, "reward_before_std": 0.5113980742171407, "reward_change_max": 0.0, "reward_change_mean": -0.2978768069297075, "reward_change_min": -0.46243468672037125, "reward_change_std": 0.18266624584794044, "reward_std": 0.560118405148387, "rewards/accuracy_reward": 0.2083333395421505, "rewards/cosine_scaled_reward": -0.0674455389380455, "step": 89 }, { "clip_fraction": 0.0, "completion_length": 2662.5416870117188, "epoch": 0.10285714285714286, "grad_norm": 0.26214170455932617, "kl": 0.0022667646408081055, "lambda_div_used": 0.5369801968336105, "learning_rate": 9.83423155058946e-07, "loss": 0.0359, "reward": -0.5144274234771729, "reward_after_mean": -0.5144274234771729, "reward_after_std": 0.24395596608519554, "reward_before_mean": -0.25238063000142574, "reward_before_std": 0.16241276543587446, "reward_change_max": 0.0, "reward_change_mean": -0.26204679161310196, "reward_change_min": -0.3946038670837879, "reward_change_std": 0.14339484833180904, "reward_std": 0.2439559753984213, "rewards/accuracy_reward": 0.0, "rewards/cosine_scaled_reward": -0.25238063000142574, "step": 90 }, { "clip_fraction": 0.0, "completion_length": 3211.7083587646484, "epoch": 0.104, "grad_norm": 0.15202385187149048, "kl": 0.0010356009006500244, "lambda_div_used": 0.5878335386514664, "learning_rate": 9.825677631722435e-07, "loss": 0.0293, "reward": -0.36480371560901403, "reward_after_mean": -0.36480371560901403, "reward_after_std": 0.48261873982846737, "reward_before_mean": -0.11783344019204378, "reward_before_std": 0.398667024448514, "reward_change_max": 0.0, "reward_change_mean": -0.24697027914226055, "reward_change_min": -0.36048777401447296, "reward_change_std": 0.13215262442827225, "reward_std": 0.4826187454164028, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.1803334429860115, "step": 91 }, { "clip_fraction": 0.0, "completion_length": 2890.812530517578, "epoch": 0.10514285714285715, "grad_norm": 0.27371829748153687, "kl": 0.0036166608333587646, "lambda_div_used": 0.58245500177145, "learning_rate": 9.816912885430258e-07, "loss": 0.0004, "reward": -0.329551937058568, "reward_after_mean": -0.329551937058568, "reward_after_std": 0.42550402879714966, "reward_before_mean": -0.057730644941329956, "reward_before_std": 0.3785192295908928, "reward_change_max": 0.0, "reward_change_mean": -0.27182128839194775, "reward_change_min": -0.4946183152496815, "reward_change_std": 0.1709953173995018, "reward_std": 0.4255040492862463, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.14106397703289986, "step": 92 }, { "clip_fraction": 0.0, "completion_length": 3483.7083435058594, "epoch": 0.10628571428571429, "grad_norm": 0.1873013973236084, "kl": 0.0013532638549804688, "lambda_div_used": 0.5382214263081551, "learning_rate": 9.807937738894303e-07, "loss": -0.0172, "reward": -0.5241896361112595, "reward_after_mean": -0.5241896361112595, "reward_after_std": 0.24293815158307552, "reward_before_mean": -0.264988811686635, "reward_before_std": 0.16782900504767895, "reward_change_max": 0.0, "reward_change_mean": -0.2592008262872696, "reward_change_min": -0.3763637840747833, "reward_change_std": 0.13893114682286978, "reward_std": 0.24293815344572067, "rewards/accuracy_reward": 0.0, "rewards/cosine_scaled_reward": -0.2649888228625059, "step": 93 }, { "clip_fraction": 0.0, "completion_length": 2981.000045776367, "epoch": 0.10742857142857143, "grad_norm": 0.21617752313613892, "kl": 0.0022999942302703857, "lambda_div_used": 0.5629464462399483, "learning_rate": 9.798752629550546e-07, "loss": -0.0327, "reward": -0.26671504601836205, "reward_after_mean": -0.26671504601836205, "reward_after_std": 0.4054209440946579, "reward_before_mean": 0.10193732008337975, "reward_before_std": 0.2889593252912164, "reward_change_max": 0.0, "reward_change_mean": -0.3686523735523224, "reward_change_min": -0.5633241385221481, "reward_change_std": 0.2078973911702633, "reward_std": 0.4054209478199482, "rewards/accuracy_reward": 0.1666666679084301, "rewards/cosine_scaled_reward": -0.0647293459624052, "step": 94 }, { "clip_fraction": 0.0, "completion_length": 3439.312530517578, "epoch": 0.10857142857142857, "grad_norm": 0.13604795932769775, "kl": 0.000415116548538208, "lambda_div_used": 0.5756574347615242, "learning_rate": 9.78935800506826e-07, "loss": 0.0262, "reward": -0.41877901554107666, "reward_after_mean": -0.41877901554107666, "reward_after_std": 0.40109362453222275, "reward_before_mean": -0.1668172136414796, "reward_before_std": 0.33935534581542015, "reward_change_max": 0.0, "reward_change_mean": -0.2519617844372988, "reward_change_min": -0.3857460096478462, "reward_change_std": 0.1420799745246768, "reward_std": 0.4010936263948679, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.2084838803857565, "step": 95 }, { "clip_fraction": 0.0, "completion_length": 2704.2708435058594, "epoch": 0.10971428571428571, "grad_norm": 0.20047779381275177, "kl": 0.0051297349855303764, "lambda_div_used": 0.6217555478215218, "learning_rate": 9.779754323328192e-07, "loss": 0.0457, "reward": -0.1323249633423984, "reward_after_mean": -0.1323249633423984, "reward_after_std": 0.5937134642153978, "reward_before_mean": 0.1469519715756178, "reward_before_std": 0.5621943678706884, "reward_change_max": 0.0, "reward_change_mean": -0.2792769465595484, "reward_change_min": -0.44912633299827576, "reward_change_std": 0.175774110481143, "reward_std": 0.5937134884297848, "rewards/accuracy_reward": 0.2083333395421505, "rewards/cosine_scaled_reward": -0.061381348059512675, "step": 96 }, { "clip_fraction": 0.0, "completion_length": 3322.9166870117188, "epoch": 0.11085714285714286, "grad_norm": 0.15216822922229767, "kl": 0.0009270659647881985, "lambda_div_used": 0.6076014935970306, "learning_rate": 9.769942052400235e-07, "loss": 0.021, "reward": -0.1877904962748289, "reward_after_mean": -0.1877904962748289, "reward_after_std": 0.5438802763819695, "reward_before_mean": 0.10107230395078659, "reward_before_std": 0.4948186855763197, "reward_change_max": 0.0, "reward_change_mean": -0.28886279836297035, "reward_change_min": -0.45709601789712906, "reward_change_std": 0.1760774115100503, "reward_std": 0.5438802968710661, "rewards/accuracy_reward": 0.2083333395421505, "rewards/cosine_scaled_reward": -0.10726103466004133, "step": 97 }, { "clip_fraction": 0.0, "completion_length": 3023.687530517578, "epoch": 0.112, "grad_norm": 0.2097436487674713, "kl": 0.000522226095199585, "lambda_div_used": 0.5420983880758286, "learning_rate": 9.759921670520634e-07, "loss": 0.0412, "reward": -0.3570786379277706, "reward_after_mean": -0.3570786379277706, "reward_after_std": 0.29755837842822075, "reward_before_mean": 0.0002752896398305893, "reward_before_std": 0.18490633368492126, "reward_change_max": 0.0, "reward_change_mean": -0.35735392570495605, "reward_change_min": -0.5063533559441566, "reward_change_std": 0.18806256912648678, "reward_std": 0.29755839332938194, "rewards/accuracy_reward": 0.125, "rewards/cosine_scaled_reward": -0.1247247289866209, "step": 98 }, { "clip_fraction": 0.0, "completion_length": 2887.4166984558105, "epoch": 0.11314285714285714, "grad_norm": 0.21412575244903564, "kl": 0.0009143352508544922, "lambda_div_used": 0.5811761543154716, "learning_rate": 9.749693666068663e-07, "loss": 0.0317, "reward": -0.2264253906905651, "reward_after_mean": -0.2264253906905651, "reward_after_std": 0.4730576742440462, "reward_before_mean": 0.13288430962711573, "reward_before_std": 0.3723627310246229, "reward_change_max": 0.0, "reward_change_mean": -0.3593096621334553, "reward_change_min": -0.554396990686655, "reward_change_std": 0.20623203460127115, "reward_std": 0.4730576779693365, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.05461570713669062, "step": 99 }, { "clip_fraction": 0.0, "completion_length": 2901.1458435058594, "epoch": 0.11428571428571428, "grad_norm": 0.1713719218969345, "kl": 0.004265546798706055, "lambda_div_used": 0.6015851572155952, "learning_rate": 9.739258537542835e-07, "loss": -0.0252, "reward": -0.10858878120779991, "reward_after_mean": -0.10858878120779991, "reward_after_std": 0.4791636262089014, "reward_before_mean": 0.22857920825481415, "reward_before_std": 0.46819167025387287, "reward_change_max": 0.0, "reward_change_mean": -0.33716796711087227, "reward_change_min": -0.5483759865164757, "reward_change_std": 0.21862902119755745, "reward_std": 0.4791636485606432, "rewards/accuracy_reward": 0.2500000074505806, "rewards/cosine_scaled_reward": -0.02142081130295992, "step": 100 }, { "clip_fraction": 0.0, "completion_length": 2973.8958435058594, "epoch": 0.11542857142857142, "grad_norm": 0.19749435782432556, "kl": 0.0016905665397644043, "lambda_div_used": 0.5860537365078926, "learning_rate": 9.728616793536587e-07, "loss": 0.0798, "reward": -0.18634457513689995, "reward_after_mean": -0.18634457513689995, "reward_after_std": 0.487332122400403, "reward_before_mean": 0.17873737215995789, "reward_before_std": 0.3957668961957097, "reward_change_max": 0.0, "reward_change_mean": -0.365081962198019, "reward_change_min": -0.6156877651810646, "reward_change_std": 0.22073478996753693, "reward_std": 0.48733213171362877, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.029595959931612015, "step": 101 }, { "clip_fraction": 0.0, "completion_length": 2879.0625610351562, "epoch": 0.11657142857142858, "grad_norm": 0.3304881453514099, "kl": 0.003381013870239258, "lambda_div_used": 0.6091511473059654, "learning_rate": 9.717768952713511e-07, "loss": 0.1102, "reward": -0.21267129853367805, "reward_after_mean": -0.21267129853367805, "reward_after_std": 0.5297415778040886, "reward_before_mean": 0.05652322247624397, "reward_before_std": 0.5078626712784171, "reward_change_max": 0.0, "reward_change_mean": -0.26919451728463173, "reward_change_min": -0.5111882165074348, "reward_change_std": 0.18575704842805862, "reward_std": 0.5297415852546692, "rewards/accuracy_reward": 0.1666666716337204, "rewards/cosine_scaled_reward": -0.11014345163857797, "step": 102 }, { "clip_fraction": 0.0, "completion_length": 3174.8958740234375, "epoch": 0.11771428571428572, "grad_norm": 0.19761358201503754, "kl": 0.0023492276668548584, "lambda_div_used": 0.6038852706551552, "learning_rate": 9.706715543782064e-07, "loss": 0.0582, "reward": -0.15667882561683655, "reward_after_mean": -0.15667882561683655, "reward_after_std": 0.5092404931783676, "reward_before_mean": 0.15082880295813084, "reward_before_std": 0.48307971749454737, "reward_change_max": 0.0, "reward_change_mean": -0.3075076453387737, "reward_change_min": -0.5402365177869797, "reward_change_std": 0.20462561398744583, "reward_std": 0.5092405062168837, "rewards/accuracy_reward": 0.1875000037252903, "rewards/cosine_scaled_reward": -0.036671207286417484, "step": 103 }, { "clip_fraction": 0.0, "completion_length": 2892.6875, "epoch": 0.11885714285714286, "grad_norm": 0.1969767063856125, "kl": 0.0022545456886291504, "lambda_div_used": 0.5755093023180962, "learning_rate": 9.695457105469804e-07, "loss": 0.0235, "reward": -0.28715056739747524, "reward_after_mean": -0.28715056739747524, "reward_after_std": 0.40210954286158085, "reward_before_mean": 0.03021797351539135, "reward_before_std": 0.341007056646049, "reward_change_max": 0.0, "reward_change_mean": -0.3173685558140278, "reward_change_min": -0.4784534201025963, "reward_change_std": 0.17809483129531145, "reward_std": 0.40210955031216145, "rewards/accuracy_reward": 0.12500000558793545, "rewards/cosine_scaled_reward": -0.09478203114122152, "step": 104 }, { "clip_fraction": 0.0, "completion_length": 2939.6041717529297, "epoch": 0.12, "grad_norm": 0.21561495959758759, "kl": 0.002199232578277588, "lambda_div_used": 0.6419632509350777, "learning_rate": 9.683994186497132e-07, "loss": 0.0624, "reward": -0.06735992804169655, "reward_after_mean": -0.06735992804169655, "reward_after_std": 0.6821530722081661, "reward_before_mean": 0.21826376300305128, "reward_before_std": 0.6594405565410852, "reward_change_max": 0.0, "reward_change_mean": -0.2856236845254898, "reward_change_min": -0.5236329026520252, "reward_change_std": 0.19643001910299063, "reward_std": 0.682153083384037, "rewards/accuracy_reward": 0.2291666716337204, "rewards/cosine_scaled_reward": -0.01090291328728199, "step": 105 }, { "clip_fraction": 0.0, "completion_length": 2443.625030517578, "epoch": 0.12114285714285715, "grad_norm": 0.3406102955341339, "kl": 0.021846607327461243, "lambda_div_used": 0.6142596453428268, "learning_rate": 9.672327345550543e-07, "loss": 0.2398, "reward": 0.2029220126569271, "reward_after_mean": 0.2029220126569271, "reward_after_std": 0.59070660546422, "reward_before_mean": 0.699662122875452, "reward_before_std": 0.5280779888853431, "reward_change_max": 0.0, "reward_change_mean": -0.49674011021852493, "reward_change_min": -0.7522984854876995, "reward_change_std": 0.30086423084139824, "reward_std": 0.5907066203653812, "rewards/accuracy_reward": 0.4583333469927311, "rewards/cosine_scaled_reward": 0.24132876843214035, "step": 106 }, { "clip_fraction": 0.0, "completion_length": 2990.645927429199, "epoch": 0.12228571428571429, "grad_norm": 0.26557278633117676, "kl": 0.001790761947631836, "lambda_div_used": 0.5581931248307228, "learning_rate": 9.66045715125541e-07, "loss": 0.0551, "reward": -0.23433807864785194, "reward_after_mean": -0.23433807864785194, "reward_after_std": 0.35804971866309643, "reward_before_mean": 0.14043704979121685, "reward_before_std": 0.2673763260245323, "reward_change_max": 0.0, "reward_change_mean": -0.3747751358896494, "reward_change_min": -0.5405581407248974, "reward_change_std": 0.2100204424932599, "reward_std": 0.3580497223883867, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.06789627112448215, "step": 107 }, { "clip_fraction": 0.0, "completion_length": 3117.7708740234375, "epoch": 0.12342857142857143, "grad_norm": 0.1650419533252716, "kl": 0.0014681816101074219, "lambda_div_used": 0.6084636449813843, "learning_rate": 9.648384182148252e-07, "loss": 0.0632, "reward": -0.21457673609256744, "reward_after_mean": -0.21457673609256744, "reward_after_std": 0.525347139686346, "reward_before_mean": 0.057420844212174416, "reward_before_std": 0.5053407922387123, "reward_change_max": 0.0, "reward_change_mean": -0.27199758775532246, "reward_change_min": -0.4748513847589493, "reward_change_std": 0.18406732566654682, "reward_std": 0.525347139686346, "rewards/accuracy_reward": 0.14583333395421505, "rewards/cosine_scaled_reward": -0.08841249160468578, "step": 108 }, { "clip_fraction": 0.0, "completion_length": 3097.3333740234375, "epoch": 0.12457142857142857, "grad_norm": 0.15857857465744019, "kl": 0.000908970832824707, "lambda_div_used": 0.576347753405571, "learning_rate": 9.636109026648554e-07, "loss": 0.0291, "reward": -0.3165609445422888, "reward_after_mean": -0.3165609445422888, "reward_after_std": 0.40352628752589226, "reward_before_mean": -0.03347342275083065, "reward_before_std": 0.3466195184737444, "reward_change_max": 0.0, "reward_change_mean": -0.2830875124782324, "reward_change_min": -0.4354559760540724, "reward_change_std": 0.16175096854567528, "reward_std": 0.40352629497647285, "rewards/accuracy_reward": 0.12500000558793545, "rewards/cosine_scaled_reward": -0.1584734208881855, "step": 109 }, { "clip_fraction": 0.0, "completion_length": 3051.8958587646484, "epoch": 0.12571428571428572, "grad_norm": 0.23943375051021576, "kl": 0.020646899938583374, "lambda_div_used": 0.6240595281124115, "learning_rate": 9.623632283030077e-07, "loss": 0.0087, "reward": -0.14237141981720924, "reward_after_mean": -0.14237141981720924, "reward_after_std": 0.5905976593494415, "reward_before_mean": 0.15653660893440247, "reward_before_std": 0.5776723609305918, "reward_change_max": 0.0, "reward_change_mean": -0.29890802316367626, "reward_change_min": -0.5991829633712769, "reward_change_std": 0.21704162284731865, "reward_std": 0.5905976612120867, "rewards/accuracy_reward": 0.1875000037252903, "rewards/cosine_scaled_reward": -0.03096340410411358, "step": 110 }, { "clip_fraction": 0.0, "completion_length": 3435.0833740234375, "epoch": 0.12685714285714286, "grad_norm": 0.16462112963199615, "kl": 0.0024847984313964844, "lambda_div_used": 0.5980314090847969, "learning_rate": 9.610954559391704e-07, "loss": 0.0343, "reward": -0.22843145579099655, "reward_after_mean": -0.22843145579099655, "reward_after_std": 0.502366553992033, "reward_before_mean": 0.06416077725589275, "reward_before_std": 0.45396197214722633, "reward_change_max": 0.0, "reward_change_mean": -0.29259222745895386, "reward_change_min": -0.4791128374636173, "reward_change_std": 0.1829609675332904, "reward_std": 0.5023665633052588, "rewards/accuracy_reward": 0.14583333395421505, "rewards/cosine_scaled_reward": -0.08167256228625774, "step": 111 }, { "clip_fraction": 0.0, "completion_length": 3356.7084045410156, "epoch": 0.128, "grad_norm": 0.15902268886566162, "kl": 0.0010784417390823364, "lambda_div_used": 0.6163632422685623, "learning_rate": 9.598076473627796e-07, "loss": 0.0256, "reward": -0.10950333066284657, "reward_after_mean": -0.10950333066284657, "reward_after_std": 0.6373277362436056, "reward_before_mean": 0.2305896393954754, "reward_before_std": 0.534846268594265, "reward_change_max": 0.0, "reward_change_mean": -0.3400929421186447, "reward_change_min": -0.5188563875854015, "reward_change_std": 0.19268847163766623, "reward_std": 0.6373277511447668, "rewards/accuracy_reward": 0.25000000186264515, "rewards/cosine_scaled_reward": -0.01941038854420185, "step": 112 }, { "clip_fraction": 0.0, "completion_length": 3272.937515258789, "epoch": 0.12914285714285714, "grad_norm": 0.18756113946437836, "kl": 0.007985830307006836, "lambda_div_used": 0.6090519055724144, "learning_rate": 9.58499865339809e-07, "loss": 0.0143, "reward": -0.20772958546876907, "reward_after_mean": -0.20772958546876907, "reward_after_std": 0.5353738944977522, "reward_before_mean": 0.0654355026781559, "reward_before_std": 0.5089074652642012, "reward_change_max": 0.0, "reward_change_mean": -0.2731650825589895, "reward_change_min": -0.5029582045972347, "reward_change_std": 0.18915251456201077, "reward_std": 0.5353739075362682, "rewards/accuracy_reward": 0.1666666679084301, "rewards/cosine_scaled_reward": -0.10123117081820965, "step": 113 }, { "clip_fraction": 0.0, "completion_length": 2939.1875534057617, "epoch": 0.13028571428571428, "grad_norm": 0.18340705335140228, "kl": 0.004691362380981445, "lambda_div_used": 0.5606123358011246, "learning_rate": 9.571721736097088e-07, "loss": -0.0151, "reward": -0.39945124462246895, "reward_after_mean": -0.39945124462246895, "reward_after_std": 0.33841873705387115, "reward_before_mean": -0.12489438056945801, "reward_before_std": 0.2770812250673771, "reward_change_max": 0.0, "reward_change_mean": -0.2745568696409464, "reward_change_min": -0.43397925049066544, "reward_change_std": 0.16178398951888084, "reward_std": 0.3384187463670969, "rewards/accuracy_reward": 0.0625, "rewards/cosine_scaled_reward": -0.18739437125623226, "step": 114 }, { "clip_fraction": 0.0, "completion_length": 3026.1458587646484, "epoch": 0.13142857142857142, "grad_norm": 0.21686024963855743, "kl": 0.0031099319458007812, "lambda_div_used": 0.594468891620636, "learning_rate": 9.55824636882301e-07, "loss": 0.0716, "reward": -0.1858784444630146, "reward_after_mean": -0.1858784444630146, "reward_after_std": 0.48573578521609306, "reward_before_mean": 0.12610083259642124, "reward_before_std": 0.43493235763162374, "reward_change_max": 0.0, "reward_change_mean": -0.3119792751967907, "reward_change_min": -0.5229315757751465, "reward_change_std": 0.1947900839149952, "reward_std": 0.48573580011725426, "rewards/accuracy_reward": 0.2291666753590107, "rewards/cosine_scaled_reward": -0.1030658520758152, "step": 115 }, { "clip_fraction": 0.0, "completion_length": 3581.5, "epoch": 0.13257142857142856, "grad_norm": 0.16519060730934143, "kl": 0.0021669864654541016, "lambda_div_used": 0.5928610190749168, "learning_rate": 9.54457320834625e-07, "loss": 0.0014, "reward": -0.3730320198228583, "reward_after_mean": -0.3730320198228583, "reward_after_std": 0.5048024039715528, "reward_before_mean": -0.14476402755826712, "reward_before_std": 0.42094550654292107, "reward_change_max": 0.0, "reward_change_mean": -0.22826798632740974, "reward_change_min": -0.3364454470574856, "reward_change_std": 0.12293815240263939, "reward_std": 0.5048024263232946, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.20726402662694454, "step": 116 }, { "clip_fraction": 0.0, "completion_length": 3417.3541870117188, "epoch": 0.1337142857142857, "grad_norm": 0.18881908059120178, "kl": 0.003082275390625, "lambda_div_used": 0.5354732051491737, "learning_rate": 9.530702921077358e-07, "loss": 0.006, "reward": -0.52970090508461, "reward_after_mean": -0.52970090508461, "reward_after_std": 0.24015513435006142, "reward_before_mean": -0.2731985878199339, "reward_before_std": 0.15570900402963161, "reward_change_max": 0.0, "reward_change_mean": -0.25650231912732124, "reward_change_min": -0.37872645631432533, "reward_change_std": 0.13538880366832018, "reward_std": 0.24015513435006142, "rewards/accuracy_reward": 0.0, "rewards/cosine_scaled_reward": -0.27319858223199844, "step": 117 }, { "clip_fraction": 0.0, "completion_length": 3217.812530517578, "epoch": 0.13485714285714287, "grad_norm": 0.22023425996303558, "kl": 0.0016322135925292969, "lambda_div_used": 0.6560031473636627, "learning_rate": 9.516636183034564e-07, "loss": 0.0866, "reward": 0.1312931291759014, "reward_after_mean": 0.1312931291759014, "reward_after_std": 0.7012096773833036, "reward_before_mean": 0.47709209844470024, "reward_before_std": 0.7338668256998062, "reward_change_max": 0.0, "reward_change_mean": -0.3457989804446697, "reward_change_min": -0.6342227831482887, "reward_change_std": 0.2567507326602936, "reward_std": 0.7012096997350454, "rewards/accuracy_reward": 0.3541666716337204, "rewards/cosine_scaled_reward": 0.12292543239891529, "step": 118 }, { "clip_fraction": 0.0, "completion_length": 2588.8541946411133, "epoch": 0.136, "grad_norm": 0.21671034395694733, "kl": 0.006726264953613281, "lambda_div_used": 0.5864673927426338, "learning_rate": 9.502373679810839e-07, "loss": -0.0143, "reward": -0.18286527134478092, "reward_after_mean": -0.18286527134478092, "reward_after_std": 0.4859150927513838, "reward_before_mean": 0.1851686891168356, "reward_before_std": 0.40001448057591915, "reward_change_max": 0.0, "reward_change_mean": -0.368033979088068, "reward_change_min": -0.6127960942685604, "reward_change_std": 0.22380509041249752, "reward_std": 0.4859151318669319, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.02316465089097619, "step": 119 }, { "clip_fraction": 0.0, "completion_length": 2857.0208740234375, "epoch": 0.13714285714285715, "grad_norm": 0.28477415442466736, "kl": 0.05393409729003906, "lambda_div_used": 0.5997197106480598, "learning_rate": 9.487916106540465e-07, "loss": -0.0772, "reward": -0.19789774296805263, "reward_after_mean": -0.19789774296805263, "reward_after_std": 0.5723651926964521, "reward_before_mean": 0.1297035962343216, "reward_before_std": 0.4560971390455961, "reward_change_max": 0.0, "reward_change_mean": -0.3276013247668743, "reward_change_min": -0.46229342743754387, "reward_change_std": 0.1723043080419302, "reward_std": 0.5723652020096779, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.057796402368694544, "step": 120 }, { "clip_fraction": 0.0, "completion_length": 2591.5625228881836, "epoch": 0.1382857142857143, "grad_norm": 0.16562210023403168, "kl": 0.007612705230712891, "lambda_div_used": 0.588602215051651, "learning_rate": 9.473264167865171e-07, "loss": -0.0097, "reward": -0.1832500956952572, "reward_after_mean": -0.1832500956952572, "reward_after_std": 0.49626706168055534, "reward_before_mean": 0.17315150797367096, "reward_before_std": 0.40745146945118904, "reward_change_max": 0.0, "reward_change_mean": -0.356401601806283, "reward_change_min": -0.5996612049639225, "reward_change_std": 0.21441051177680492, "reward_std": 0.49626707285642624, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.03518182039260864, "step": 121 }, { "clip_fraction": 0.0, "completion_length": 3201.7083740234375, "epoch": 0.13942857142857143, "grad_norm": 0.2166738212108612, "kl": 0.0031104087829589844, "lambda_div_used": 0.5987401753664017, "learning_rate": 9.458418577899774e-07, "loss": 0.0516, "reward": -0.039603039622306824, "reward_after_mean": -0.039603039622306824, "reward_after_std": 0.47058134339749813, "reward_before_mean": 0.32864847779273987, "reward_before_std": 0.4533402416855097, "reward_change_max": 0.0, "reward_change_mean": -0.36825149320065975, "reward_change_min": -0.5661437995731831, "reward_change_std": 0.22861475870013237, "reward_std": 0.4705813527107239, "rewards/accuracy_reward": 0.2708333432674408, "rewards/cosine_scaled_reward": 0.05781510937958956, "step": 122 }, { "clip_fraction": 0.0, "completion_length": 3152.229248046875, "epoch": 0.14057142857142857, "grad_norm": 0.15404550731182098, "kl": 0.002484560012817383, "lambda_div_used": 0.5779790133237839, "learning_rate": 9.443380060197385e-07, "loss": -0.0081, "reward": -0.20699449256062508, "reward_after_mean": -0.20699449256062508, "reward_after_std": 0.3932098187506199, "reward_before_mean": 0.12272489443421364, "reward_before_std": 0.35472595132887363, "reward_change_max": 0.0, "reward_change_mean": -0.3297193981707096, "reward_change_min": -0.5193503461778164, "reward_change_std": 0.19783841259777546, "reward_std": 0.3932098299264908, "rewards/accuracy_reward": 0.1875000074505806, "rewards/cosine_scaled_reward": -0.06477508880198002, "step": 123 }, { "clip_fraction": 0.0, "completion_length": 2887.958396911621, "epoch": 0.1417142857142857, "grad_norm": 0.19075822830200195, "kl": 0.24529361724853516, "lambda_div_used": 0.6367463618516922, "learning_rate": 9.428149347714143e-07, "loss": -0.0091, "reward": -0.043416159227490425, "reward_after_mean": -0.043416159227490425, "reward_after_std": 0.6688170228153467, "reward_before_mean": 0.266201576218009, "reward_before_std": 0.6342105437070131, "reward_change_max": 0.0, "reward_change_mean": -0.3096177503466606, "reward_change_min": -0.5306770168244839, "reward_change_std": 0.20241453684866428, "reward_std": 0.6688170302659273, "rewards/accuracy_reward": 0.2500000074505806, "rewards/cosine_scaled_reward": 0.01620158093282953, "step": 124 }, { "clip_fraction": 0.0, "completion_length": 2935.500015258789, "epoch": 0.14285714285714285, "grad_norm": 0.16372336447238922, "kl": 0.0892171859741211, "lambda_div_used": 0.608710303902626, "learning_rate": 9.412727182773486e-07, "loss": 0.0004, "reward": -0.1132714906707406, "reward_after_mean": -0.1132714906707406, "reward_after_std": 0.5880292318761349, "reward_before_mean": 0.23416612297296524, "reward_before_std": 0.5022992007434368, "reward_change_max": 0.0, "reward_change_mean": -0.34743762016296387, "reward_change_min": -0.5739596225321293, "reward_change_std": 0.20932148303836584, "reward_std": 0.5880292393267155, "rewards/accuracy_reward": 0.25000000186264515, "rewards/cosine_scaled_reward": -0.015833888202905655, "step": 125 }, { "clip_fraction": 0.0, "completion_length": 2946.000030517578, "epoch": 0.144, "grad_norm": 0.19263476133346558, "kl": 0.001348733901977539, "lambda_div_used": 0.5741089954972267, "learning_rate": 9.397114317029974e-07, "loss": 0.037, "reward": -0.24507060274481773, "reward_after_mean": -0.24507060274481773, "reward_after_std": 0.3843264617025852, "reward_before_mean": 0.07774791494011879, "reward_before_std": 0.33831747248768806, "reward_change_max": 0.0, "reward_change_mean": -0.3228185139596462, "reward_change_min": -0.5080043151974678, "reward_change_std": 0.1930001936852932, "reward_std": 0.3843264728784561, "rewards/accuracy_reward": 0.1666666716337204, "rewards/cosine_scaled_reward": -0.0889187604188919, "step": 126 }, { "clip_fraction": 0.0, "completion_length": 3321.2291870117188, "epoch": 0.14514285714285713, "grad_norm": 0.21647047996520996, "kl": 0.0024771690368652344, "lambda_div_used": 0.5367048606276512, "learning_rate": 9.381311511432658e-07, "loss": -0.0263, "reward": -0.5233126580715179, "reward_after_mean": -0.5233126580715179, "reward_after_std": 0.24122811667621136, "reward_before_mean": -0.25869173742830753, "reward_before_std": 0.1611304171383381, "reward_change_max": 0.0, "reward_change_mean": -0.26462091132998466, "reward_change_min": -0.3869423381984234, "reward_change_std": 0.14034162927418947, "reward_std": 0.2412281259894371, "rewards/accuracy_reward": 0.0, "rewards/cosine_scaled_reward": -0.25869173742830753, "step": 127 }, { "clip_fraction": 0.0, "completion_length": 3082.812530517578, "epoch": 0.1462857142857143, "grad_norm": 0.1646365523338318, "kl": 0.00429379940032959, "lambda_div_used": 0.6071840301156044, "learning_rate": 9.36531953618799e-07, "loss": 0.0491, "reward": 0.005701523274183273, "reward_after_mean": 0.005701523274183273, "reward_after_std": 0.5655696392059326, "reward_before_mean": 0.4053007960319519, "reward_before_std": 0.49548310600221157, "reward_change_max": 0.0, "reward_change_mean": -0.39959926530718803, "reward_change_min": -0.6030891872942448, "reward_change_std": 0.238234281539917, "reward_std": 0.5655696503818035, "rewards/accuracy_reward": 0.31250000558793545, "rewards/cosine_scaled_reward": 0.09280077531002462, "step": 128 }, { "clip_fraction": 0.0, "completion_length": 3554.0208435058594, "epoch": 0.14742857142857144, "grad_norm": 0.14819103479385376, "kl": 0.0030744075775146484, "lambda_div_used": 0.5695768147706985, "learning_rate": 9.34913917072228e-07, "loss": 0.0139, "reward": -0.3974629668518901, "reward_after_mean": -0.3974629668518901, "reward_after_std": 0.40156686678528786, "reward_before_mean": -0.1324129467830062, "reward_before_std": 0.3131661769002676, "reward_change_max": 0.0, "reward_change_mean": -0.26505002193152905, "reward_change_min": -0.38079287111759186, "reward_change_std": 0.1410841178148985, "reward_std": 0.40156687796115875, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.1740796146914363, "step": 129 }, { "clip_fraction": 0.0, "completion_length": 3407.1666870117188, "epoch": 0.14857142857142858, "grad_norm": 0.16314037144184113, "kl": 0.008331298828125, "lambda_div_used": 0.5569034516811371, "learning_rate": 9.332771203643714e-07, "loss": 0.0113, "reward": -0.3312334306538105, "reward_after_mean": -0.3312334306538105, "reward_after_std": 0.30558963119983673, "reward_before_mean": -0.006856339983642101, "reward_before_std": 0.25353086180984974, "reward_change_max": 0.0, "reward_change_mean": -0.32437711395323277, "reward_change_min": -0.48944010585546494, "reward_change_std": 0.18379169888794422, "reward_std": 0.3055896405130625, "rewards/accuracy_reward": 0.1041666716337204, "rewards/cosine_scaled_reward": -0.11102299485355616, "step": 130 }, { "clip_fraction": 0.0, "completion_length": 2991.437545776367, "epoch": 0.14971428571428572, "grad_norm": 0.20164915919303894, "kl": 0.0241243839263916, "lambda_div_used": 0.6288070753216743, "learning_rate": 9.316216432703916e-07, "loss": 0.0059, "reward": 0.011569222435355186, "reward_after_mean": 0.011569222435355186, "reward_after_std": 0.6672684717923403, "reward_before_mean": 0.37154264003038406, "reward_before_std": 0.6025523003190756, "reward_change_max": 0.0, "reward_change_mean": -0.3599734287708998, "reward_change_min": -0.5932495594024658, "reward_change_std": 0.23335123620927334, "reward_std": 0.6672684978693724, "rewards/accuracy_reward": 0.33333333395421505, "rewards/cosine_scaled_reward": 0.038209314458072186, "step": 131 }, { "clip_fraction": 0.0, "completion_length": 3395.5833740234375, "epoch": 0.15085714285714286, "grad_norm": 0.1822521835565567, "kl": 0.0032308101654052734, "lambda_div_used": 0.6204090118408203, "learning_rate": 9.299475664759068e-07, "loss": 0.0361, "reward": -0.18310143752023578, "reward_after_mean": -0.18310143752023578, "reward_after_std": 0.5879561994224787, "reward_before_mean": 0.08326816186308861, "reward_before_std": 0.5585347628220916, "reward_change_max": 0.0, "reward_change_mean": -0.26636960357427597, "reward_change_min": -0.485028225928545, "reward_change_std": 0.1797371245920658, "reward_std": 0.5879562236368656, "rewards/accuracy_reward": 0.20833334140479565, "rewards/cosine_scaled_reward": -0.12506516324356198, "step": 132 }, { "clip_fraction": 0.0, "completion_length": 3423.6875610351562, "epoch": 0.152, "grad_norm": 0.1939377635717392, "kl": 0.0032417774200439453, "lambda_div_used": 0.5426758751273155, "learning_rate": 9.282549715730579e-07, "loss": -0.0059, "reward": -0.42481518629938364, "reward_after_mean": -0.42481518629938364, "reward_after_std": 0.27526283264160156, "reward_before_mean": -0.11293287388980389, "reward_before_std": 0.18761167209595442, "reward_change_max": 0.0, "reward_change_mean": -0.31188230961561203, "reward_change_min": -0.45120835676789284, "reward_change_std": 0.16707892064005136, "reward_std": 0.2752628345042467, "rewards/accuracy_reward": 0.1041666716337204, "rewards/cosine_scaled_reward": -0.21709954645484686, "step": 133 }, { "clip_fraction": 0.0, "completion_length": 3022.7291870117188, "epoch": 0.15314285714285714, "grad_norm": 0.206606924533844, "kl": 0.03278493881225586, "lambda_div_used": 0.6046839281916618, "learning_rate": 9.265439410565328e-07, "loss": 0.0341, "reward": -0.07414344511926174, "reward_after_mean": -0.07414344511926174, "reward_after_std": 0.5560939908027649, "reward_before_mean": 0.2953918972052634, "reward_before_std": 0.48563989251852036, "reward_change_max": 0.0, "reward_change_mean": -0.3695353548973799, "reward_change_min": -0.5760605782270432, "reward_change_std": 0.22501015942543745, "reward_std": 0.5560940019786358, "rewards/accuracy_reward": 0.29166666977107525, "rewards/cosine_scaled_reward": 0.00372522696852684, "step": 134 }, { "clip_fraction": 0.0, "completion_length": 2443.750072479248, "epoch": 0.15428571428571428, "grad_norm": 0.3037879765033722, "kl": 0.013937950134277344, "lambda_div_used": 0.616770051419735, "learning_rate": 9.248145583195447e-07, "loss": 0.191, "reward": 0.17862435802817345, "reward_after_mean": 0.17862435802817345, "reward_after_std": 0.5873858220875263, "reward_before_mean": 0.6582186818122864, "reward_before_std": 0.5401020906865597, "reward_change_max": 0.0, "reward_change_mean": -0.4795943293720484, "reward_change_min": -0.7301103062927723, "reward_change_std": 0.2966425511986017, "reward_std": 0.5873858444392681, "rewards/accuracy_reward": 0.4583333432674408, "rewards/cosine_scaled_reward": 0.19988534040749073, "step": 135 }, { "clip_fraction": 0.0, "completion_length": 3418.250030517578, "epoch": 0.15542857142857142, "grad_norm": 0.17704758048057556, "kl": 0.0062084197998046875, "lambda_div_used": 0.6302180886268616, "learning_rate": 9.230669076497687e-07, "loss": 0.0283, "reward": -0.021340223029255867, "reward_after_mean": -0.021340223029255867, "reward_after_std": 0.6091137193143368, "reward_before_mean": 0.3001672737300396, "reward_before_std": 0.605611389502883, "reward_change_max": 0.0, "reward_change_mean": -0.32150749303400517, "reward_change_min": -0.5832352973520756, "reward_change_std": 0.22389173042029142, "reward_std": 0.6091137453913689, "rewards/accuracy_reward": 0.2708333395421505, "rewards/cosine_scaled_reward": 0.029333939775824547, "step": 136 }, { "clip_fraction": 0.0, "completion_length": 3519.125, "epoch": 0.15657142857142858, "grad_norm": 0.14247606694698334, "kl": 0.004940986633300781, "lambda_div_used": 0.5559564679861069, "learning_rate": 9.213010742252327e-07, "loss": -0.0033, "reward": -0.4558934085071087, "reward_after_mean": -0.4558934085071087, "reward_after_std": 0.3285232465714216, "reward_before_mean": -0.18790924572385848, "reward_before_std": 0.25175026152282953, "reward_change_max": 0.0, "reward_change_mean": -0.267984164878726, "reward_change_min": -0.38948308303952217, "reward_change_std": 0.1436179345473647, "reward_std": 0.3285232614725828, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.20874257944524288, "step": 137 }, { "clip_fraction": 0.0, "completion_length": 3022.250015258789, "epoch": 0.15771428571428572, "grad_norm": 0.1857508420944214, "kl": 0.003807544708251953, "lambda_div_used": 0.5555576086044312, "learning_rate": 9.195171441101668e-07, "loss": 0.0236, "reward": -0.3569270521402359, "reward_after_mean": -0.3569270521402359, "reward_after_std": 0.30704532004892826, "reward_before_mean": -0.047900162637233734, "reward_before_std": 0.24731899984180927, "reward_change_max": 0.0, "reward_change_mean": -0.30902687832713127, "reward_change_min": -0.4676646441221237, "reward_change_std": 0.17348954733461142, "reward_std": 0.30704533867537975, "rewards/accuracy_reward": 0.1041666716337204, "rewards/cosine_scaled_reward": -0.15206682868301868, "step": 138 }, { "clip_fraction": 0.0, "completion_length": 3358.7291870117188, "epoch": 0.15885714285714286, "grad_norm": 0.16412624716758728, "kl": 0.0035724639892578125, "lambda_div_used": 0.6047067120671272, "learning_rate": 9.177152042508077e-07, "loss": 0.0697, "reward": -0.3240318773314357, "reward_after_mean": -0.3240318773314357, "reward_after_std": 0.5254602003842592, "reward_before_mean": -0.08715378493070602, "reward_before_std": 0.48024341836571693, "reward_change_max": 0.0, "reward_change_mean": -0.23687809705734253, "reward_change_min": -0.42061349749565125, "reward_change_std": 0.14863354805856943, "reward_std": 0.5254602301865816, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.17048711562529206, "step": 139 }, { "clip_fraction": 0.0, "completion_length": 3379.1666870117188, "epoch": 0.16, "grad_norm": 0.19232088327407837, "kl": 0.01123046875, "lambda_div_used": 0.5895432904362679, "learning_rate": 9.158953424711624e-07, "loss": -0.0135, "reward": -0.24200669303536415, "reward_after_mean": -0.24200669303536415, "reward_after_std": 0.5481930077075958, "reward_before_mean": 0.09891421953216195, "reward_before_std": 0.40956787252798676, "reward_change_max": 0.0, "reward_change_mean": -0.3409209195524454, "reward_change_min": -0.472070824354887, "reward_change_std": 0.17763086315244436, "reward_std": 0.5481930207461119, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.08858578093349934, "step": 140 }, { "clip_fraction": 0.0, "completion_length": 3333.479217529297, "epoch": 0.16114285714285714, "grad_norm": 0.17990702390670776, "kl": 0.005831718444824219, "lambda_div_used": 0.6218598261475563, "learning_rate": 9.140576474687263e-07, "loss": 0.0315, "reward": -0.2724424530752003, "reward_after_mean": -0.2724424530752003, "reward_after_std": 0.5944670829921961, "reward_before_mean": -0.03745802119374275, "reward_before_std": 0.5648406567052007, "reward_change_max": 0.0, "reward_change_mean": -0.23498443141579628, "reward_change_min": -0.47893765568733215, "reward_change_std": 0.16880445461720228, "reward_std": 0.5944670960307121, "rewards/accuracy_reward": 0.14583333767950535, "rewards/cosine_scaled_reward": -0.1832913690013811, "step": 141 }, { "clip_fraction": 0.0, "completion_length": 3229.0208740234375, "epoch": 0.16228571428571428, "grad_norm": 0.29604029655456543, "kl": 0.006026744842529297, "lambda_div_used": 0.6064464673399925, "learning_rate": 9.122022088101613e-07, "loss": 0.0109, "reward": -0.16769713163375854, "reward_after_mean": -0.16769713163375854, "reward_after_std": 0.511599974706769, "reward_before_mean": 0.1383611150085926, "reward_before_std": 0.4924522005021572, "reward_change_max": 0.0, "reward_change_mean": -0.30605825409293175, "reward_change_min": -0.5262043438851833, "reward_change_std": 0.20410850830376148, "reward_std": 0.5115999896079302, "rewards/accuracy_reward": 0.2083333395421505, "rewards/cosine_scaled_reward": -0.06997222313657403, "step": 142 }, { "clip_fraction": 0.0, "completion_length": 3281.0833740234375, "epoch": 0.16342857142857142, "grad_norm": 0.1977819949388504, "kl": 0.09860420227050781, "lambda_div_used": 0.5833587870001793, "learning_rate": 9.103291169269299e-07, "loss": 0.0859, "reward": -0.36300591891631484, "reward_after_mean": -0.36300591891631484, "reward_after_std": 0.43086348101496696, "reward_before_mean": -0.10792777501046658, "reward_before_std": 0.38438355596736073, "reward_change_max": 0.0, "reward_change_mean": -0.2550781387835741, "reward_change_min": -0.4078437276184559, "reward_change_std": 0.15707087609916925, "reward_std": 0.4308634866029024, "rewards/accuracy_reward": 0.08333333395421505, "rewards/cosine_scaled_reward": -0.19126112153753638, "step": 143 }, { "clip_fraction": 0.0, "completion_length": 3215.0833587646484, "epoch": 0.16457142857142856, "grad_norm": 0.1647995561361313, "kl": 0.008655548095703125, "lambda_div_used": 0.6045641824603081, "learning_rate": 9.084384631108882e-07, "loss": -0.0186, "reward": -0.19354566000401974, "reward_after_mean": -0.19354566000401974, "reward_after_std": 0.5126497764140368, "reward_before_mean": 0.09084580233320594, "reward_before_std": 0.4810970528051257, "reward_change_max": 0.0, "reward_change_mean": -0.28439145162701607, "reward_change_min": -0.46073341369628906, "reward_change_std": 0.18039765022695065, "reward_std": 0.5126497894525528, "rewards/accuracy_reward": 0.18750000558793545, "rewards/cosine_scaled_reward": -0.09665422141551971, "step": 144 }, { "clip_fraction": 0.0, "completion_length": 2694.750015258789, "epoch": 0.1657142857142857, "grad_norm": 0.2760597765445709, "kl": 0.006956815719604492, "lambda_div_used": 0.5869975462555885, "learning_rate": 9.065303395098358e-07, "loss": -0.0599, "reward": -0.08179877698421478, "reward_after_mean": -0.08179877698421478, "reward_after_std": 0.5045480858534575, "reward_before_mean": 0.3235731632448733, "reward_before_std": 0.39837881876155734, "reward_change_max": 0.0, "reward_change_mean": -0.4053719528019428, "reward_change_min": -0.5960347391664982, "reward_change_std": 0.23043952323496342, "reward_std": 0.5045481082051992, "rewards/accuracy_reward": 0.31250000558793545, "rewards/cosine_scaled_reward": 0.011073160916566849, "step": 145 }, { "clip_fraction": 0.0, "completion_length": 3212.562530517578, "epoch": 0.16685714285714287, "grad_norm": 0.19820109009742737, "kl": 0.0034825801849365234, "lambda_div_used": 0.573294386267662, "learning_rate": 9.046048391230247e-07, "loss": 0.0164, "reward": -0.43352778162807226, "reward_after_mean": -0.43352778162807226, "reward_after_std": 0.40275699831545353, "reward_before_mean": -0.1929878443479538, "reward_before_std": 0.330263695679605, "reward_change_max": 0.0, "reward_change_mean": -0.2405399437993765, "reward_change_min": -0.37657370418310165, "reward_change_std": 0.13261152990162373, "reward_std": 0.402757003903389, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.2346545048058033, "step": 146 }, { "clip_fraction": 0.0, "completion_length": 3465.979217529297, "epoch": 0.168, "grad_norm": 0.1675552874803543, "kl": 0.006984233856201172, "lambda_div_used": 0.6165526434779167, "learning_rate": 9.026620557966279e-07, "loss": 0.0435, "reward": -0.24559049191884696, "reward_after_mean": -0.24559049191884696, "reward_after_std": 0.5987250991165638, "reward_before_mean": -0.003920666873455048, "reward_before_std": 0.5365209635347128, "reward_change_max": 0.0, "reward_change_mean": -0.24166982993483543, "reward_change_min": -0.39302265271544456, "reward_change_std": 0.1459288764744997, "reward_std": 0.5987251158803701, "rewards/accuracy_reward": 0.12500000186264515, "rewards/cosine_scaled_reward": -0.12892067292705178, "step": 147 }, { "clip_fraction": 0.0, "completion_length": 3040.104217529297, "epoch": 0.16914285714285715, "grad_norm": 0.16194044053554535, "kl": 0.010385513305664062, "lambda_div_used": 0.5780738145112991, "learning_rate": 9.007020842191634e-07, "loss": -0.0128, "reward": -0.24759145267307758, "reward_after_mean": -0.24759145267307758, "reward_after_std": 0.47053366526961327, "reward_before_mean": 0.1050882339477539, "reward_before_std": 0.3541037440299988, "reward_change_max": 0.0, "reward_change_mean": -0.3526796866208315, "reward_change_min": -0.5096157230436802, "reward_change_std": 0.18679230101406574, "reward_std": 0.4705336671322584, "rewards/accuracy_reward": 0.1666666679084301, "rewards/cosine_scaled_reward": -0.061578433495014906, "step": 148 }, { "clip_fraction": 0.0, "completion_length": 3171.5833740234375, "epoch": 0.1702857142857143, "grad_norm": 0.14514388144016266, "kl": 0.004037141799926758, "lambda_div_used": 0.5804071500897408, "learning_rate": 8.987250199168808e-07, "loss": 0.0011, "reward": -0.13505330216139555, "reward_after_mean": -0.13505330216139555, "reward_after_std": 0.45987606048583984, "reward_before_mean": 0.2546880329027772, "reward_before_std": 0.36488907039165497, "reward_change_max": 0.0, "reward_change_mean": -0.38974130153656006, "reward_change_min": -0.564285933971405, "reward_change_std": 0.2169877402484417, "reward_std": 0.4598760772496462, "rewards/accuracy_reward": 0.25000000558793545, "rewards/cosine_scaled_reward": 0.004688006825745106, "step": 149 }, { "clip_fraction": 0.0, "completion_length": 3237.9583740234375, "epoch": 0.17142857142857143, "grad_norm": 0.1528966873884201, "kl": 0.011098861694335938, "lambda_div_used": 0.6220898926258087, "learning_rate": 8.967309592491052e-07, "loss": 0.0413, "reward": -0.2357348818331957, "reward_after_mean": -0.2357348818331957, "reward_after_std": 0.6165088415145874, "reward_before_mean": 0.003244686871767044, "reward_before_std": 0.5655408930033445, "reward_change_max": 0.0, "reward_change_mean": -0.23897957988083363, "reward_change_min": -0.44260188937187195, "reward_change_std": 0.15603933855891228, "reward_std": 0.616508848965168, "rewards/accuracy_reward": 0.1250000037252903, "rewards/cosine_scaled_reward": -0.12175530241802335, "step": 150 }, { "clip_fraction": 0.0, "completion_length": 3297.3958740234375, "epoch": 0.17257142857142857, "grad_norm": 0.16029568016529083, "kl": 0.00822591781616211, "lambda_div_used": 0.5745164379477501, "learning_rate": 8.9471999940354e-07, "loss": 0.014, "reward": -0.15804946795105934, "reward_after_mean": -0.15804946795105934, "reward_after_std": 0.4451928697526455, "reward_before_mean": 0.23821012489497662, "reward_before_std": 0.33810407761484385, "reward_change_max": 0.0, "reward_change_mean": -0.39625960774719715, "reward_change_min": -0.5654826126992702, "reward_change_std": 0.21966782212257385, "reward_std": 0.4451928809285164, "rewards/accuracy_reward": 0.22916666977107525, "rewards/cosine_scaled_reward": 0.009043465368449688, "step": 151 }, { "clip_fraction": 0.0, "completion_length": 3060.8333435058594, "epoch": 0.1737142857142857, "grad_norm": 0.25289276242256165, "kl": 0.007596492767333984, "lambda_div_used": 0.5495098456740379, "learning_rate": 8.926922383915315e-07, "loss": 0.0104, "reward": -0.4798025581985712, "reward_after_mean": -0.4798025581985712, "reward_after_std": 0.29928642325103283, "reward_before_mean": -0.21831904165446758, "reward_before_std": 0.22063454845920205, "reward_change_max": 0.0, "reward_change_mean": -0.2614835239946842, "reward_change_min": -0.3717127852141857, "reward_change_std": 0.1392099717631936, "reward_std": 0.2992864269763231, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.23915237560868263, "step": 152 }, { "clip_fraction": 0.0, "completion_length": 3259.5833740234375, "epoch": 0.17485714285714285, "grad_norm": 0.213038370013237, "kl": 0.009700775146484375, "lambda_div_used": 0.5835977271199226, "learning_rate": 8.906477750432903e-07, "loss": 0.0433, "reward": -0.3274975838139653, "reward_after_mean": -0.3274975838139653, "reward_after_std": 0.4250865038484335, "reward_before_mean": -0.05488423630595207, "reward_before_std": 0.38222964480519295, "reward_change_max": 0.0, "reward_change_mean": -0.2726133484393358, "reward_change_min": -0.48245660215616226, "reward_change_std": 0.1732556838542223, "reward_std": 0.42508652433753014, "rewards/accuracy_reward": 0.1041666679084301, "rewards/cosine_scaled_reward": -0.1590509070083499, "step": 153 }, { "clip_fraction": 0.0, "completion_length": 3558.104217529297, "epoch": 0.176, "grad_norm": 0.1455099880695343, "kl": 0.0033369064331054688, "lambda_div_used": 0.6287582516670227, "learning_rate": 8.88586709003076e-07, "loss": 0.0133, "reward": -0.14025272196158767, "reward_after_mean": -0.14025272196158767, "reward_after_std": 0.6268025431782007, "reward_before_mean": 0.12356582563370466, "reward_before_std": 0.6016785530373454, "reward_change_max": 0.0, "reward_change_mean": -0.2638185489922762, "reward_change_min": -0.44624603912234306, "reward_change_std": 0.17495773546397686, "reward_std": 0.6268025785684586, "rewards/accuracy_reward": 0.1875000037252903, "rewards/cosine_scaled_reward": -0.06393416714854538, "step": 154 }, { "clip_fraction": 0.0, "completion_length": 3103.666717529297, "epoch": 0.17714285714285713, "grad_norm": 0.18942059576511383, "kl": 0.008495330810546875, "lambda_div_used": 0.5698634833097458, "learning_rate": 8.865091407243394e-07, "loss": 0.0257, "reward": -0.032269224524497986, "reward_after_mean": -0.032269224524497986, "reward_after_std": 0.4197168964892626, "reward_before_mean": 0.4300368260592222, "reward_before_std": 0.3127097301185131, "reward_change_max": 0.0, "reward_change_mean": -0.4623060114681721, "reward_change_min": -0.6557185426354408, "reward_change_std": 0.2544031012803316, "reward_std": 0.41971689835190773, "rewards/accuracy_reward": 0.3333333432674408, "rewards/cosine_scaled_reward": 0.09670346044003963, "step": 155 }, { "clip_fraction": 0.0, "completion_length": 3575.9791870117188, "epoch": 0.1782857142857143, "grad_norm": 0.14295417070388794, "kl": 0.004237174987792969, "lambda_div_used": 0.6037876382470131, "learning_rate": 8.844151714648274e-07, "loss": 0.0014, "reward": -0.2925546169281006, "reward_after_mean": -0.2925546169281006, "reward_after_std": 0.5249353367835283, "reward_before_mean": -0.038088101893663406, "reward_before_std": 0.47752281837165356, "reward_change_max": 0.0, "reward_change_mean": -0.2544665280729532, "reward_change_min": -0.41321099549531937, "reward_change_std": 0.15618102066218853, "reward_std": 0.5249353460967541, "rewards/accuracy_reward": 0.1041666679084301, "rewards/cosine_scaled_reward": -0.1422547702677548, "step": 156 }, { "clip_fraction": 0.0, "completion_length": 3323.6666870117188, "epoch": 0.17942857142857144, "grad_norm": 0.17017501592636108, "kl": 0.006339550018310547, "lambda_div_used": 0.557308703660965, "learning_rate": 8.823049032816478e-07, "loss": 0.0136, "reward": -0.32858159579336643, "reward_after_mean": -0.32858159579336643, "reward_after_std": 0.37395724654197693, "reward_before_mean": 0.011270022951066494, "reward_before_std": 0.2547361049801111, "reward_change_max": 0.0, "reward_change_mean": -0.3398516271263361, "reward_change_min": -0.48356175422668457, "reward_change_std": 0.1784765599295497, "reward_std": 0.3739572502672672, "rewards/accuracy_reward": 0.14583333395421505, "rewards/cosine_scaled_reward": -0.13456329703330994, "step": 157 }, { "clip_fraction": 0.0, "completion_length": 3456.1250610351562, "epoch": 0.18057142857142858, "grad_norm": 0.17302246391773224, "kl": 0.003945350646972656, "lambda_div_used": 0.6036131903529167, "learning_rate": 8.801784390262943e-07, "loss": 0.0055, "reward": -0.05629691109061241, "reward_after_mean": -0.05629691109061241, "reward_after_std": 0.5824002176523209, "reward_before_mean": 0.33863569144159555, "reward_before_std": 0.4843088276684284, "reward_change_max": 0.0, "reward_change_mean": -0.39493261836469173, "reward_change_min": -0.6200560890138149, "reward_change_std": 0.23636492900550365, "reward_std": 0.582400219514966, "rewards/accuracy_reward": 0.27083333767950535, "rewards/cosine_scaled_reward": 0.06780234724283218, "step": 158 }, { "clip_fraction": 0.0, "completion_length": 3532.9791870117188, "epoch": 0.18171428571428572, "grad_norm": 0.13993459939956665, "kl": 0.008701324462890625, "lambda_div_used": 0.5565592646598816, "learning_rate": 8.780358823396352e-07, "loss": 0.0254, "reward": -0.4379761107265949, "reward_after_mean": -0.4379761107265949, "reward_after_std": 0.3240527082234621, "reward_before_mean": -0.17050177045166492, "reward_before_std": 0.25670385733246803, "reward_change_max": 0.0, "reward_change_mean": -0.26747431978583336, "reward_change_min": -0.4343995824456215, "reward_change_std": 0.15539650060236454, "reward_std": 0.32405271753668785, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.21216844581067562, "step": 159 }, { "clip_fraction": 0.0, "completion_length": 3370.437530517578, "epoch": 0.18285714285714286, "grad_norm": 0.17298126220703125, "kl": 0.008138656616210938, "lambda_div_used": 0.5891826152801514, "learning_rate": 8.758773376468604e-07, "loss": 0.0383, "reward": -0.16474956646561623, "reward_after_mean": -0.16474956646561623, "reward_after_std": 0.4341984763741493, "reward_before_mean": 0.16870688181370497, "reward_before_std": 0.40600887034088373, "reward_change_max": 0.0, "reward_change_mean": -0.3334564808756113, "reward_change_min": -0.5314052142202854, "reward_change_std": 0.20622879173606634, "reward_std": 0.43419847823679447, "rewards/accuracy_reward": 0.2291666753590107, "rewards/cosine_scaled_reward": -0.06045978702604771, "step": 160 }, { "clip_fraction": 0.0, "completion_length": 3307.3958435058594, "epoch": 0.184, "grad_norm": 0.172169491648674, "kl": 0.012319564819335938, "lambda_div_used": 0.5956699401140213, "learning_rate": 8.737029101523929e-07, "loss": 0.0222, "reward": -0.19135635159909725, "reward_after_mean": -0.19135635159909725, "reward_after_std": 0.5530895814299583, "reward_before_mean": 0.14698314014822245, "reward_before_std": 0.43515510112047195, "reward_change_max": 0.0, "reward_change_mean": -0.338339488953352, "reward_change_min": -0.48101864010095596, "reward_change_std": 0.17739303410053253, "reward_std": 0.5530895907431841, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.04051686264574528, "step": 161 }, { "clip_fraction": 0.0, "completion_length": 3498.250030517578, "epoch": 0.18514285714285714, "grad_norm": 0.1709449589252472, "kl": 0.009785652160644531, "lambda_div_used": 0.611346609890461, "learning_rate": 8.715127058347614e-07, "loss": 0.0279, "reward": -0.23669173195958138, "reward_after_mean": -0.23669173195958138, "reward_after_std": 0.5506523419171572, "reward_before_mean": 0.022784698754549026, "reward_before_std": 0.5215989332646132, "reward_change_max": 0.0, "reward_change_mean": -0.25947642885148525, "reward_change_min": -0.5050196386873722, "reward_change_std": 0.1816305760294199, "reward_std": 0.5506523586809635, "rewards/accuracy_reward": 0.1458333358168602, "rewards/cosine_scaled_reward": -0.12304865010082722, "step": 162 }, { "clip_fraction": 0.0, "completion_length": 2789.812530517578, "epoch": 0.18628571428571428, "grad_norm": 0.18061158061027527, "kl": 0.008713722229003906, "lambda_div_used": 0.5765003487467766, "learning_rate": 8.693068314414344e-07, "loss": 0.0746, "reward": -0.0672077015042305, "reward_after_mean": -0.0672077015042305, "reward_after_std": 0.43557936511933804, "reward_before_mean": 0.35438246466219425, "reward_before_std": 0.3476250753737986, "reward_change_max": 0.0, "reward_change_mean": -0.42159019596874714, "reward_change_min": -0.6456112191081047, "reward_change_std": 0.23985794186592102, "reward_std": 0.43557936884462833, "rewards/accuracy_reward": 0.3333333358168602, "rewards/cosine_scaled_reward": 0.021049125120043755, "step": 163 }, { "clip_fraction": 0.0, "completion_length": 2931.291679382324, "epoch": 0.18742857142857142, "grad_norm": 0.20612357556819916, "kl": 0.10277938842773438, "lambda_div_used": 0.5614631026983261, "learning_rate": 8.670853944836176e-07, "loss": -0.0404, "reward": -0.25212905183434486, "reward_after_mean": -0.25212905183434486, "reward_after_std": 0.39177498035132885, "reward_before_mean": 0.12557442858815193, "reward_before_std": 0.28237397503107786, "reward_change_max": 0.0, "reward_change_mean": -0.377703458070755, "reward_change_min": -0.5706961192190647, "reward_change_std": 0.21161762066185474, "reward_std": 0.391774982213974, "rewards/accuracy_reward": 0.1666666679084301, "rewards/cosine_scaled_reward": -0.04109225235879421, "step": 164 }, { "clip_fraction": 0.0, "completion_length": 3417.9791870117188, "epoch": 0.18857142857142858, "grad_norm": 0.16617462038993835, "kl": 0.008676528930664062, "lambda_div_used": 0.5787941217422485, "learning_rate": 8.648485032310144e-07, "loss": -0.0087, "reward": -0.298875629901886, "reward_after_mean": -0.298875629901886, "reward_after_std": 0.4017774127423763, "reward_before_mean": -0.003390883095562458, "reward_before_std": 0.3545601461082697, "reward_change_max": 0.0, "reward_change_mean": -0.2954847402870655, "reward_change_min": -0.4517757259309292, "reward_change_std": 0.17376293614506721, "reward_std": 0.40177742950618267, "rewards/accuracy_reward": 0.12500000558793545, "rewards/cosine_scaled_reward": -0.12839087742031552, "step": 165 }, { "clip_fraction": 0.0, "completion_length": 3473.8333740234375, "epoch": 0.18971428571428572, "grad_norm": 0.14783978462219238, "kl": 0.003978729248046875, "lambda_div_used": 0.6031280383467674, "learning_rate": 8.625962667065487e-07, "loss": 0.0383, "reward": -0.21845939941704273, "reward_after_mean": -0.21845939941704273, "reward_after_std": 0.5085912439972162, "reward_before_mean": 0.07111049693776295, "reward_before_std": 0.47475082986056805, "reward_change_max": 0.0, "reward_change_mean": -0.2895698957145214, "reward_change_min": -0.49096018821001053, "reward_change_std": 0.1821102648973465, "reward_std": 0.5085912570357323, "rewards/accuracy_reward": 0.16666667349636555, "rewards/cosine_scaled_reward": -0.09555616229772568, "step": 166 }, { "clip_fraction": 0.0, "completion_length": 3031.041732788086, "epoch": 0.19085714285714286, "grad_norm": 0.1715136170387268, "kl": 0.038117408752441406, "lambda_div_used": 0.5885661765933037, "learning_rate": 8.603287946810513e-07, "loss": 0.083, "reward": -0.2653086595237255, "reward_after_mean": -0.2653086595237255, "reward_after_std": 0.43174828588962555, "reward_before_mean": 0.02882637269794941, "reward_before_std": 0.4080026801675558, "reward_change_max": 0.0, "reward_change_mean": -0.2941350396722555, "reward_change_min": -0.4922946132719517, "reward_change_std": 0.1907910518348217, "reward_std": 0.431748291477561, "rewards/accuracy_reward": 0.125, "rewards/cosine_scaled_reward": -0.09617362171411514, "step": 167 }, { "clip_fraction": 0.0, "completion_length": 3187.4583587646484, "epoch": 0.192, "grad_norm": 0.17107972502708435, "kl": 0.004605293273925781, "lambda_div_used": 0.6155762076377869, "learning_rate": 8.580461976679099e-07, "loss": -0.0257, "reward": -0.15246706921607256, "reward_after_mean": -0.15246706921607256, "reward_after_std": 0.5662419721484184, "reward_before_mean": 0.15201041847467422, "reward_before_std": 0.5325471749529243, "reward_change_max": 0.0, "reward_change_mean": -0.3044775053858757, "reward_change_min": -0.5361337065696716, "reward_change_std": 0.2014566957950592, "reward_std": 0.5662419833242893, "rewards/accuracy_reward": 0.18750000558793545, "rewards/cosine_scaled_reward": -0.035489581525325775, "step": 168 }, { "clip_fraction": 0.0, "completion_length": 3034.1458740234375, "epoch": 0.19314285714285714, "grad_norm": 0.15446673333644867, "kl": 0.0060253143310546875, "lambda_div_used": 0.5790615305304527, "learning_rate": 8.557485869176825e-07, "loss": -0.0029, "reward": 0.14061698503792286, "reward_after_mean": 0.14061698503792286, "reward_after_std": 0.5485950075089931, "reward_before_mean": 0.6988185402005911, "reward_before_std": 0.3581995740532875, "reward_change_max": 0.0, "reward_change_mean": -0.5582015290856361, "reward_change_min": -0.7899037674069405, "reward_change_std": 0.2953033493831754, "reward_std": 0.5485950279980898, "rewards/accuracy_reward": 0.5000000055879354, "rewards/cosine_scaled_reward": 0.19881849735975266, "step": 169 }, { "clip_fraction": 0.0, "completion_length": 2951.875030517578, "epoch": 0.19428571428571428, "grad_norm": 0.16040825843811035, "kl": 0.016638755798339844, "lambda_div_used": 0.560679629445076, "learning_rate": 8.534360744126753e-07, "loss": 0.0435, "reward": -0.17987253330647945, "reward_after_mean": -0.17987253330647945, "reward_after_std": 0.43466968461871147, "reward_before_mean": 0.2520832261070609, "reward_before_std": 0.27346578426659107, "reward_change_max": 0.0, "reward_change_mean": -0.4319557659327984, "reward_change_min": -0.6064207814633846, "reward_change_std": 0.22567898873239756, "reward_std": 0.4346696902066469, "rewards/accuracy_reward": 0.27083333395421505, "rewards/cosine_scaled_reward": -0.018750112503767014, "step": 170 }, { "clip_fraction": 0.0, "completion_length": 3269.041702270508, "epoch": 0.19542857142857142, "grad_norm": 0.16092920303344727, "kl": 0.004572391510009766, "lambda_div_used": 0.6146296262741089, "learning_rate": 8.511087728614862e-07, "loss": 0.0178, "reward": -0.12626560777425766, "reward_after_mean": -0.12626560777425766, "reward_after_std": 0.5721440799534321, "reward_before_mean": 0.18320718850009143, "reward_before_std": 0.5280091362074018, "reward_change_max": 0.0, "reward_change_mean": -0.30947281420230865, "reward_change_min": -0.4679437726736069, "reward_change_std": 0.18679437600076199, "reward_std": 0.5721440874040127, "rewards/accuracy_reward": 0.2291666753590107, "rewards/cosine_scaled_reward": -0.04595948662608862, "step": 171 }, { "clip_fraction": 0.0, "completion_length": 3103.2083587646484, "epoch": 0.19657142857142856, "grad_norm": 0.18477661907672882, "kl": 0.006927490234375, "lambda_div_used": 0.5566391721367836, "learning_rate": 8.487667956935087e-07, "loss": 0.0021, "reward": -0.16334839165210724, "reward_after_mean": -0.16334839165210724, "reward_after_std": 0.4313113037496805, "reward_before_mean": 0.28386795427650213, "reward_before_std": 0.25497481785714626, "reward_change_max": 0.0, "reward_change_mean": -0.44721634685993195, "reward_change_min": -0.6107739247381687, "reward_change_std": 0.22866252530366182, "reward_std": 0.43131132796406746, "rewards/accuracy_reward": 0.27083333395421505, "rewards/cosine_scaled_reward": 0.013034614268690348, "step": 172 }, { "clip_fraction": 0.0, "completion_length": 2637.0625228881836, "epoch": 0.1977142857142857, "grad_norm": 0.22738754749298096, "kl": 0.016168594360351562, "lambda_div_used": 0.6129439249634743, "learning_rate": 8.464102570534061e-07, "loss": 0.0577, "reward": -0.28153929114341736, "reward_after_mean": -0.28153929114341736, "reward_after_std": 0.5640011355280876, "reward_before_mean": -0.03958232072182, "reward_before_std": 0.5151211321353912, "reward_change_max": 0.0, "reward_change_mean": -0.24195699580013752, "reward_change_min": -0.4067324548959732, "reward_change_std": 0.15156757924705744, "reward_std": 0.5640011541545391, "rewards/accuracy_reward": 0.12500000186264515, "rewards/cosine_scaled_reward": -0.16458232887089252, "step": 173 }, { "clip_fraction": 0.0, "completion_length": 3253.4166870117188, "epoch": 0.19885714285714284, "grad_norm": 0.17404116690158844, "kl": 0.008798599243164062, "lambda_div_used": 0.5919044315814972, "learning_rate": 8.440392717955475e-07, "loss": 0.0174, "reward": -0.21083719469606876, "reward_after_mean": -0.21083719469606876, "reward_after_std": 0.536060806363821, "reward_before_mean": 0.12530302570667118, "reward_before_std": 0.4163727965205908, "reward_change_max": 0.0, "reward_change_mean": -0.33614021725952625, "reward_change_min": -0.48955636844038963, "reward_change_std": 0.1774530140683055, "reward_std": 0.5360608138144016, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.062197002582252026, "step": 174 }, { "clip_fraction": 0.0, "completion_length": 2978.1667098999023, "epoch": 0.2, "grad_norm": 0.18372827768325806, "kl": 0.0071868896484375, "lambda_div_used": 0.6068883538246155, "learning_rate": 8.416539554784089e-07, "loss": -0.0043, "reward": -0.0928361751139164, "reward_after_mean": -0.0928361751139164, "reward_after_std": 0.5076029077172279, "reward_before_mean": 0.24198724888265133, "reward_before_std": 0.49847083911299706, "reward_change_max": 0.0, "reward_change_mean": -0.3348234295845032, "reward_change_min": -0.5438389666378498, "reward_change_std": 0.22006033454090357, "reward_std": 0.5076029095798731, "rewards/accuracy_reward": 0.2500000074505806, "rewards/cosine_scaled_reward": -0.00801275484263897, "step": 175 }, { "clip_fraction": 0.0, "completion_length": 3104.187530517578, "epoch": 0.20114285714285715, "grad_norm": 0.21719031035900116, "kl": 0.0113677978515625, "lambda_div_used": 0.6736999601125717, "learning_rate": 8.392544243589427e-07, "loss": 0.0562, "reward": 0.09344447404146194, "reward_after_mean": 0.09344447404146194, "reward_after_std": 0.8027716539800167, "reward_before_mean": 0.3930630199611187, "reward_before_std": 0.8141509592533112, "reward_change_max": 0.0, "reward_change_mean": -0.29961856454610825, "reward_change_min": -0.5923384893685579, "reward_change_std": 0.22679476160556078, "reward_std": 0.8027716688811779, "rewards/accuracy_reward": 0.31250000558793545, "rewards/cosine_scaled_reward": 0.0805630087852478, "step": 176 }, { "clip_fraction": 0.0, "completion_length": 3356.1458740234375, "epoch": 0.2022857142857143, "grad_norm": 0.1844586431980133, "kl": 0.009817123413085938, "lambda_div_used": 0.585597425699234, "learning_rate": 8.368407953869103e-07, "loss": -0.0153, "reward": -0.2503715232014656, "reward_after_mean": -0.2503715232014656, "reward_after_std": 0.4280230049043894, "reward_before_mean": 0.04593953117728233, "reward_before_std": 0.3930676504969597, "reward_change_max": 0.0, "reward_change_mean": -0.29631106927990913, "reward_change_min": -0.4923434183001518, "reward_change_std": 0.18544270005077124, "reward_std": 0.4280230049043894, "rewards/accuracy_reward": 0.1458333358168602, "rewards/cosine_scaled_reward": -0.09989380091428757, "step": 177 }, { "clip_fraction": 0.0, "completion_length": 3192.9583435058594, "epoch": 0.20342857142857143, "grad_norm": 0.16537626087665558, "kl": 0.0206146240234375, "lambda_div_used": 0.611080139875412, "learning_rate": 8.344131861991828e-07, "loss": 0.0055, "reward": -0.022929655387997627, "reward_after_mean": -0.022929655387997627, "reward_after_std": 0.5769748371094465, "reward_before_mean": 0.3566122278571129, "reward_before_std": 0.5182048566639423, "reward_change_max": 0.0, "reward_change_mean": -0.37954190373420715, "reward_change_min": -0.6234683394432068, "reward_change_std": 0.2423446010798216, "reward_std": 0.5769748520106077, "rewards/accuracy_reward": 0.3125000037252903, "rewards/cosine_scaled_reward": 0.04411221295595169, "step": 178 }, { "clip_fraction": 0.0, "completion_length": 3271.250030517578, "epoch": 0.20457142857142857, "grad_norm": 0.1591142863035202, "kl": 0.005904197692871094, "lambda_div_used": 0.5725052133202553, "learning_rate": 8.319717151140072e-07, "loss": 0.0071, "reward": -0.437593562528491, "reward_after_mean": -0.437593562528491, "reward_after_std": 0.4073327034711838, "reward_before_mean": -0.19368988322094083, "reward_before_std": 0.3279479709453881, "reward_change_max": 0.0, "reward_change_mean": -0.24390367232263088, "reward_change_min": -0.3698204904794693, "reward_change_std": 0.13269466254860163, "reward_std": 0.4073327202349901, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.23535654554143548, "step": 179 }, { "clip_fraction": 0.0, "completion_length": 2915.541717529297, "epoch": 0.2057142857142857, "grad_norm": 0.18673957884311676, "kl": 0.01320648193359375, "lambda_div_used": 0.6204877272248268, "learning_rate": 8.295165011252396e-07, "loss": 0.0465, "reward": 0.07505011186003685, "reward_after_mean": 0.07505011186003685, "reward_after_std": 0.6184922568500042, "reward_before_mean": 0.4863988235592842, "reward_before_std": 0.55762043222785, "reward_change_max": 0.0, "reward_change_mean": -0.411348694935441, "reward_change_min": -0.6864441372454166, "reward_change_std": 0.2617882778868079, "reward_std": 0.6184922661632299, "rewards/accuracy_reward": 0.3750000074505806, "rewards/cosine_scaled_reward": 0.111398802138865, "step": 180 }, { "clip_fraction": 0.0, "completion_length": 3251.125, "epoch": 0.20685714285714285, "grad_norm": 0.17590658366680145, "kl": 0.007853507995605469, "lambda_div_used": 0.555995024740696, "learning_rate": 8.270476638965461e-07, "loss": -0.0023, "reward": -0.31235363334417343, "reward_after_mean": -0.31235363334417343, "reward_after_std": 0.384617256000638, "reward_before_mean": 0.0376875763759017, "reward_before_std": 0.25200952496379614, "reward_change_max": 0.0, "reward_change_mean": -0.3500411994755268, "reward_change_min": -0.4919729493558407, "reward_change_std": 0.1814344823360443, "reward_std": 0.38461725786328316, "rewards/accuracy_reward": 0.14583333395421505, "rewards/cosine_scaled_reward": -0.10814575664699078, "step": 181 }, { "clip_fraction": 0.0, "completion_length": 3105.979202270508, "epoch": 0.208, "grad_norm": 0.15009306371212006, "kl": 0.008333206176757812, "lambda_div_used": 0.6555697843432426, "learning_rate": 8.245653237555705e-07, "loss": 0.0185, "reward": -0.013118128292262554, "reward_after_mean": -0.013118128292262554, "reward_after_std": 0.7568989247083664, "reward_before_mean": 0.270064536947757, "reward_before_std": 0.7185448948293924, "reward_change_max": 0.0, "reward_change_mean": -0.2831826340407133, "reward_change_min": -0.44287747144699097, "reward_change_std": 0.17636196687817574, "reward_std": 0.7568989545106888, "rewards/accuracy_reward": 0.2708333432674408, "rewards/cosine_scaled_reward": -0.0007688300684094429, "step": 182 }, { "clip_fraction": 0.0, "completion_length": 3209.041717529297, "epoch": 0.20914285714285713, "grad_norm": 0.19809211790561676, "kl": 0.01428985595703125, "lambda_div_used": 0.6128125712275505, "learning_rate": 8.220696016880687e-07, "loss": 0.0235, "reward": -0.051329316571354866, "reward_after_mean": -0.051329316571354866, "reward_after_std": 0.5420395694673061, "reward_before_mean": 0.2882953006774187, "reward_before_std": 0.5169597007334232, "reward_change_max": 0.0, "reward_change_mean": -0.3396245986223221, "reward_change_min": -0.5454091764986515, "reward_change_std": 0.2122363829985261, "reward_std": 0.5420395843684673, "rewards/accuracy_reward": 0.27083334513008595, "rewards/cosine_scaled_reward": 0.017461951822042465, "step": 183 }, { "clip_fraction": 0.0, "completion_length": 3258.6041717529297, "epoch": 0.2102857142857143, "grad_norm": 0.20414941012859344, "kl": 0.0068187713623046875, "lambda_div_used": 0.5394841656088829, "learning_rate": 8.195606193320136e-07, "loss": -0.0208, "reward": -0.34632613882422447, "reward_after_mean": -0.34632613882422447, "reward_after_std": 0.28760869428515434, "reward_before_mean": 0.02480493299663067, "reward_before_std": 0.17338278330862522, "reward_change_max": 0.0, "reward_change_mean": -0.3711310997605324, "reward_change_min": -0.5309014357626438, "reward_change_std": 0.19529772363603115, "reward_std": 0.2876086961477995, "rewards/accuracy_reward": 0.125, "rewards/cosine_scaled_reward": -0.10019506141543388, "step": 184 }, { "clip_fraction": 0.0, "completion_length": 3437.9791870117188, "epoch": 0.21142857142857144, "grad_norm": 0.18736158311367035, "kl": 0.011694908142089844, "lambda_div_used": 0.5765019282698631, "learning_rate": 8.170384989716657e-07, "loss": 0.0319, "reward": -0.31583110243082047, "reward_after_mean": -0.31583110243082047, "reward_after_std": 0.3850574642419815, "reward_before_mean": -0.013598954305052757, "reward_before_std": 0.3435060679912567, "reward_change_max": 0.0, "reward_change_mean": -0.30223214998841286, "reward_change_min": -0.4576663225889206, "reward_change_std": 0.17472553998231888, "reward_std": 0.3850574865937233, "rewards/accuracy_reward": 0.12500000558793545, "rewards/cosine_scaled_reward": -0.13859895430505276, "step": 185 }, { "clip_fraction": 0.0, "completion_length": 3434.6041870117188, "epoch": 0.21257142857142858, "grad_norm": 0.15987509489059448, "kl": 0.005828857421875, "lambda_div_used": 0.6050910204648972, "learning_rate": 8.145033635316128e-07, "loss": 0.0217, "reward": -0.12493636086583138, "reward_after_mean": -0.12493636086583138, "reward_after_std": 0.5005011279135942, "reward_before_mean": 0.19533411413431168, "reward_before_std": 0.48508089035749435, "reward_change_max": 0.0, "reward_change_mean": -0.32027046382427216, "reward_change_min": -0.5284873284399509, "reward_change_std": 0.21141480654478073, "reward_std": 0.5005011409521103, "rewards/accuracy_reward": 0.2291666716337204, "rewards/cosine_scaled_reward": -0.03383256494998932, "step": 186 }, { "clip_fraction": 0.0, "completion_length": 3389.7500610351562, "epoch": 0.21371428571428572, "grad_norm": 0.1716122180223465, "kl": 0.013996124267578125, "lambda_div_used": 0.592347152531147, "learning_rate": 8.119553365707802e-07, "loss": 0.0218, "reward": -0.29605016484856606, "reward_after_mean": -0.29605016484856606, "reward_after_std": 0.4498199373483658, "reward_before_mean": -0.021247809752821922, "reward_before_std": 0.4267402421683073, "reward_change_max": 0.0, "reward_change_mean": -0.2748023308813572, "reward_change_min": -0.4955432265996933, "reward_change_std": 0.18287630565464497, "reward_std": 0.44981994666159153, "rewards/accuracy_reward": 0.1041666679084301, "rewards/cosine_scaled_reward": -0.12541447952389717, "step": 187 }, { "clip_fraction": 0.0, "completion_length": 3557.7083435058594, "epoch": 0.21485714285714286, "grad_norm": 0.1581345647573471, "kl": 0.006466865539550781, "lambda_div_used": 0.5812888965010643, "learning_rate": 8.093945422764069e-07, "loss": 0.008, "reward": -0.3078702676575631, "reward_after_mean": -0.3078702676575631, "reward_after_std": 0.43122144043445587, "reward_before_mean": -0.024701565504074097, "reward_before_std": 0.3718807250261307, "reward_change_max": 0.0, "reward_change_mean": -0.28316869772970676, "reward_change_min": -0.4488030672073364, "reward_change_std": 0.16667864192277193, "reward_std": 0.4312214460223913, "rewards/accuracy_reward": 0.08333333395421505, "rewards/cosine_scaled_reward": -0.1080348901450634, "step": 188 }, { "clip_fraction": 0.0, "completion_length": 3198.9375, "epoch": 0.216, "grad_norm": 0.19062311947345734, "kl": 0.0067043304443359375, "lambda_div_used": 0.5797726735472679, "learning_rate": 8.068211054579943e-07, "loss": 0.0432, "reward": -0.19141792878508568, "reward_after_mean": -0.19141792878508568, "reward_after_std": 0.4567198269069195, "reward_before_mean": 0.1747444081120193, "reward_before_std": 0.36336799710989, "reward_change_max": 0.0, "reward_change_mean": -0.3661623205989599, "reward_change_min": -0.5529600977897644, "reward_change_std": 0.2088340139016509, "reward_std": 0.45671984925866127, "rewards/accuracy_reward": 0.20833333395421505, "rewards/cosine_scaled_reward": -0.03358893468976021, "step": 189 }, { "clip_fraction": 0.0, "completion_length": 3156.312530517578, "epoch": 0.21714285714285714, "grad_norm": 0.1547778993844986, "kl": 0.009029388427734375, "lambda_div_used": 0.5868588760495186, "learning_rate": 8.04235151541222e-07, "loss": 0.0332, "reward": -0.12204607389867306, "reward_after_mean": -0.12204607389867306, "reward_after_std": 0.48561155796051025, "reward_before_mean": 0.25893391110002995, "reward_before_std": 0.4019195716828108, "reward_change_max": 0.0, "reward_change_mean": -0.38097996823489666, "reward_change_min": -0.6010153330862522, "reward_change_std": 0.22699050419032574, "reward_std": 0.48561158776283264, "rewards/accuracy_reward": 0.2500000037252903, "rewards/cosine_scaled_reward": 0.00893389992415905, "step": 190 }, { "clip_fraction": 0.0, "completion_length": 3052.375, "epoch": 0.21828571428571428, "grad_norm": 0.1710629016160965, "kl": 0.03849029541015625, "lambda_div_used": 0.5837063267827034, "learning_rate": 8.01636806561836e-07, "loss": 0.03, "reward": -0.18280289694666862, "reward_after_mean": -0.18280289694666862, "reward_after_std": 0.4787737503647804, "reward_before_mean": 0.18333169259130955, "reward_before_std": 0.38826378528028727, "reward_change_max": 0.0, "reward_change_mean": -0.3661345764994621, "reward_change_min": -0.6050428301095963, "reward_change_std": 0.2193511137738824, "reward_std": 0.4787737689912319, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.025001641362905502, "step": 191 }, { "clip_fraction": 0.0, "completion_length": 3491.5625610351562, "epoch": 0.21942857142857142, "grad_norm": 0.1521444171667099, "kl": 0.00489044189453125, "lambda_div_used": 0.5761173516511917, "learning_rate": 7.990261971595048e-07, "loss": 0.0137, "reward": -0.3847157470881939, "reward_after_mean": -0.3847157470881939, "reward_after_std": 0.4221804942935705, "reward_before_mean": -0.12879220861941576, "reward_before_std": 0.343238091096282, "reward_change_max": 0.0, "reward_change_mean": -0.25592354126274586, "reward_change_min": -0.39532124623656273, "reward_change_std": 0.14258022606372833, "reward_std": 0.4221805166453123, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.19129220442846417, "step": 192 }, { "clip_fraction": 0.0, "completion_length": 3399.7291870117188, "epoch": 0.22057142857142858, "grad_norm": 0.1678982526063919, "kl": 0.011871337890625, "lambda_div_used": 0.5486155077815056, "learning_rate": 7.964034505716476e-07, "loss": -0.0224, "reward": -0.5118280202150345, "reward_after_mean": -0.5118280202150345, "reward_after_std": 0.2837069649249315, "reward_before_mean": -0.2657251376658678, "reward_before_std": 0.21402649395167828, "reward_change_max": 0.0, "reward_change_mean": -0.24610287882387638, "reward_change_min": -0.384936586022377, "reward_change_std": 0.13594305887818336, "reward_std": 0.28370697796344757, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.28655847534537315, "step": 193 }, { "clip_fraction": 0.0, "completion_length": 3293.6875610351562, "epoch": 0.22171428571428572, "grad_norm": 0.15788386762142181, "kl": 0.0053234100341796875, "lambda_div_used": 0.6021404713392258, "learning_rate": 7.93768694627233e-07, "loss": 0.0429, "reward": 0.011035603703930974, "reward_after_mean": 0.011035603703930974, "reward_after_std": 0.5462123416364193, "reward_before_mean": 0.42999077402055264, "reward_before_std": 0.469467893242836, "reward_change_max": 0.0, "reward_change_mean": -0.41895517334342003, "reward_change_min": -0.6308071911334991, "reward_change_std": 0.24560540914535522, "reward_std": 0.5462123528122902, "rewards/accuracy_reward": 0.31250000558793545, "rewards/cosine_scaled_reward": 0.1174907828681171, "step": 194 }, { "clip_fraction": 0.0, "completion_length": 3390.062530517578, "epoch": 0.22285714285714286, "grad_norm": 0.1422133892774582, "kl": 0.005446434020996094, "lambda_div_used": 0.5912721157073975, "learning_rate": 7.911220577405484e-07, "loss": 0.0192, "reward": -0.26650910172611475, "reward_after_mean": -0.26650910172611475, "reward_after_std": 0.4766275975853205, "reward_before_mean": 0.017198730260133743, "reward_before_std": 0.41556220687925816, "reward_change_max": 0.0, "reward_change_mean": -0.2837078347802162, "reward_change_min": -0.4380440339446068, "reward_change_std": 0.1631055912002921, "reward_std": 0.4766276068985462, "rewards/accuracy_reward": 0.1250000037252903, "rewards/cosine_scaled_reward": -0.10780127299949527, "step": 195 }, { "clip_fraction": 0.0, "completion_length": 3496.437530517578, "epoch": 0.224, "grad_norm": 0.15927068889141083, "kl": 0.004856109619140625, "lambda_div_used": 0.5781458765268326, "learning_rate": 7.884636689049422e-07, "loss": 0.024, "reward": -0.3891096324659884, "reward_after_mean": -0.3891096324659884, "reward_after_std": 0.41204992309212685, "reward_before_mean": -0.12819884344935417, "reward_before_std": 0.3546639531850815, "reward_change_max": 0.0, "reward_change_mean": -0.2609107866883278, "reward_change_min": -0.4393681026995182, "reward_change_std": 0.15767684020102024, "reward_std": 0.4120499361306429, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.19069885043427348, "step": 196 }, { "clip_fraction": 0.0, "completion_length": 3047.0208435058594, "epoch": 0.22514285714285714, "grad_norm": 0.19822825491428375, "kl": 0.009440422058105469, "lambda_div_used": 0.6301422268152237, "learning_rate": 7.857936576865356e-07, "loss": 0.0206, "reward": 0.06319928867742419, "reward_after_mean": 0.06319928867742419, "reward_after_std": 0.7072618864476681, "reward_before_mean": 0.45930443704128265, "reward_before_std": 0.5992524065077305, "reward_change_max": 0.0, "reward_change_mean": -0.39610512740910053, "reward_change_min": -0.6013565212488174, "reward_change_std": 0.2268814854323864, "reward_std": 0.7072619162499905, "rewards/accuracy_reward": 0.33333334140479565, "rewards/cosine_scaled_reward": 0.1259710679296404, "step": 197 }, { "clip_fraction": 0.0, "completion_length": 3127.562530517578, "epoch": 0.22628571428571428, "grad_norm": 0.1753966361284256, "kl": 0.011265754699707031, "lambda_div_used": 0.5487275719642639, "learning_rate": 7.831121542179086e-07, "loss": -0.0017, "reward": -0.1960901990532875, "reward_after_mean": -0.1960901990532875, "reward_after_std": 0.3432629946619272, "reward_before_mean": 0.23978716507554054, "reward_before_std": 0.21459459606558084, "reward_change_max": 0.0, "reward_change_mean": -0.4358773920685053, "reward_change_min": -0.6082254163920879, "reward_change_std": 0.23057555593550205, "reward_std": 0.34326300770044327, "rewards/accuracy_reward": 0.25, "rewards/cosine_scaled_reward": -0.010212846100330353, "step": 198 }, { "clip_fraction": 0.0, "completion_length": 3578.3333435058594, "epoch": 0.22742857142857142, "grad_norm": 0.15651564300060272, "kl": 0.009883880615234375, "lambda_div_used": 0.5590720996260643, "learning_rate": 7.804192891917571e-07, "loss": 0.0028, "reward": -0.4229575805366039, "reward_after_mean": -0.4229575805366039, "reward_after_std": 0.3386622183024883, "reward_before_mean": -0.14809601288288832, "reward_before_std": 0.2688621198758483, "reward_change_max": 0.0, "reward_change_mean": -0.27486156672239304, "reward_change_min": -0.4392392747104168, "reward_change_std": 0.1593855945393443, "reward_std": 0.3386622183024883, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.18976268218830228, "step": 199 }, { "clip_fraction": 0.0, "completion_length": 2876.8333740234375, "epoch": 0.22857142857142856, "grad_norm": 0.15169818699359894, "kl": 0.005579948425292969, "lambda_div_used": 0.6219591945409775, "learning_rate": 7.777151938545235e-07, "loss": 0.0356, "reward": 0.009905596263706684, "reward_after_mean": 0.009905596263706684, "reward_after_std": 0.6353256944566965, "reward_before_mean": 0.394716169917956, "reward_before_std": 0.5624312367290258, "reward_change_max": 0.0, "reward_change_mean": -0.38481059670448303, "reward_change_min": -0.5820214636623859, "reward_change_std": 0.23000475112348795, "reward_std": 0.6353257019072771, "rewards/accuracy_reward": 0.3333333395421505, "rewards/cosine_scaled_reward": 0.06138285622000694, "step": 200 }, { "clip_fraction": 0.0, "completion_length": 3247.0208740234375, "epoch": 0.2297142857142857, "grad_norm": 0.1644308865070343, "kl": 0.030424118041992188, "lambda_div_used": 0.6551110595464706, "learning_rate": 7.75e-07, "loss": -0.0168, "reward": 0.012024200521409512, "reward_after_mean": 0.012024200521409512, "reward_after_std": 0.7667502593249083, "reward_before_mean": 0.3109226562664844, "reward_before_std": 0.7096804045140743, "reward_change_max": 0.0, "reward_change_mean": -0.298898434266448, "reward_change_min": -0.46923108398914337, "reward_change_std": 0.18142160773277283, "reward_std": 0.7667502742260695, "rewards/accuracy_reward": 0.27083334140479565, "rewards/cosine_scaled_reward": 0.04008930642157793, "step": 201 }, { "clip_fraction": 0.0, "completion_length": 2739.2083740234375, "epoch": 0.23085714285714284, "grad_norm": 0.2415701448917389, "kl": 0.011205673217773438, "lambda_div_used": 0.5984915345907211, "learning_rate": 7.72273839962904e-07, "loss": 0.0837, "reward": 0.17140387371182442, "reward_after_mean": 0.17140387371182442, "reward_after_std": 0.5660825241357088, "reward_before_mean": 0.6926137004047632, "reward_before_std": 0.45300711458548903, "reward_change_max": 0.0, "reward_change_mean": -0.5212098266929388, "reward_change_min": -0.7827027887105942, "reward_change_std": 0.3030433254316449, "reward_std": 0.5660825371742249, "rewards/accuracy_reward": 0.4791666753590107, "rewards/cosine_scaled_reward": 0.21344703622162342, "step": 202 }, { "clip_fraction": 0.0, "completion_length": 3367.9166717529297, "epoch": 0.232, "grad_norm": 0.1717255860567093, "kl": 0.007135868072509766, "lambda_div_used": 0.5585653409361839, "learning_rate": 7.695368466124296e-07, "loss": -0.0101, "reward": -0.30565002001821995, "reward_after_mean": -0.30565002001821995, "reward_after_std": 0.37882242910563946, "reward_before_mean": 0.044357829727232456, "reward_before_std": 0.2621347298845649, "reward_change_max": 0.0, "reward_change_mean": -0.3500078674405813, "reward_change_min": -0.5026260353624821, "reward_change_std": 0.183785954490304, "reward_std": 0.37882243655622005, "rewards/accuracy_reward": 0.14583333395421505, "rewards/cosine_scaled_reward": -0.10147551540285349, "step": 203 }, { "clip_fraction": 0.0, "completion_length": 3041.250030517578, "epoch": 0.23314285714285715, "grad_norm": 0.19485335052013397, "kl": 0.028145790100097656, "lambda_div_used": 0.5799565613269806, "learning_rate": 7.667891533457718e-07, "loss": 0.0674, "reward": -0.2680759150534868, "reward_after_mean": -0.2680759150534868, "reward_after_std": 0.3968199472874403, "reward_before_mean": 0.04271291010081768, "reward_before_std": 0.3647980890236795, "reward_change_max": 0.0, "reward_change_mean": -0.31078883074223995, "reward_change_min": -0.5118796229362488, "reward_change_std": 0.19273801613599062, "reward_std": 0.3968199472874403, "rewards/accuracy_reward": 0.1458333395421505, "rewards/cosine_scaled_reward": -0.10312043130397797, "step": 204 }, { "clip_fraction": 0.0, "completion_length": 3177.500030517578, "epoch": 0.2342857142857143, "grad_norm": 0.17805610597133636, "kl": 0.0060253143310546875, "lambda_div_used": 0.6402464285492897, "learning_rate": 7.640308940816239e-07, "loss": 0.0419, "reward": 0.030928824096918106, "reward_after_mean": 0.030928824096918106, "reward_after_std": 0.6491842567920685, "reward_before_mean": 0.36594330007210374, "reward_before_std": 0.6503786351531744, "reward_change_max": 0.0, "reward_change_mean": -0.3350144773721695, "reward_change_min": -0.5699316002428532, "reward_change_std": 0.23023241478949785, "reward_std": 0.6491842679679394, "rewards/accuracy_reward": 0.33333334140479565, "rewards/cosine_scaled_reward": 0.032609956339001656, "step": 205 }, { "clip_fraction": 0.0, "completion_length": 3495.5000610351562, "epoch": 0.23542857142857143, "grad_norm": 0.1559228152036667, "kl": 0.010587692260742188, "lambda_div_used": 0.5791562497615814, "learning_rate": 7.612622032536507e-07, "loss": 0.0051, "reward": -0.42463689111173153, "reward_after_mean": -0.42463689111173153, "reward_after_std": 0.42784578539431095, "reward_before_mean": -0.1896086442284286, "reward_before_std": 0.35911196656525135, "reward_change_max": 0.0, "reward_change_mean": -0.23502822779119015, "reward_change_min": -0.3677822910249233, "reward_change_std": 0.1309291934594512, "reward_std": 0.4278458170592785, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.231275319121778, "step": 206 }, { "clip_fraction": 0.0, "completion_length": 3520.500030517578, "epoch": 0.23657142857142857, "grad_norm": 0.18334217369556427, "kl": 0.013598442077636719, "lambda_div_used": 0.5795475244522095, "learning_rate": 7.584832158039378e-07, "loss": 0.0128, "reward": -0.33587903063744307, "reward_after_mean": -0.33587903063744307, "reward_after_std": 0.4282269310206175, "reward_before_mean": -0.06741932407021523, "reward_before_std": 0.36571289878338575, "reward_change_max": 0.0, "reward_change_mean": -0.2684597074985504, "reward_change_min": -0.41530318185687065, "reward_change_std": 0.15815377235412598, "reward_std": 0.4282269552350044, "rewards/accuracy_reward": 0.08333333395421505, "rewards/cosine_scaled_reward": -0.15075265802443027, "step": 207 }, { "clip_fraction": 0.0, "completion_length": 3099.250030517578, "epoch": 0.2377142857142857, "grad_norm": 0.19052401185035706, "kl": 0.007171630859375, "lambda_div_used": 0.6005219295620918, "learning_rate": 7.556940671764124e-07, "loss": 0.0206, "reward": -0.17955744388746098, "reward_after_mean": -0.17955744388746098, "reward_after_std": 0.5050753485411406, "reward_before_mean": 0.1300262869335711, "reward_before_std": 0.4632161222398281, "reward_change_max": 0.0, "reward_change_mean": -0.30958373099565506, "reward_change_min": -0.4937345050275326, "reward_change_std": 0.1900632008910179, "reward_std": 0.505075367167592, "rewards/accuracy_reward": 0.16666667349636555, "rewards/cosine_scaled_reward": -0.03664038656279445, "step": 208 }, { "clip_fraction": 0.0, "completion_length": 3138.270866394043, "epoch": 0.23885714285714285, "grad_norm": 0.16932177543640137, "kl": 0.06981277465820312, "lambda_div_used": 0.5998589172959328, "learning_rate": 7.528948933102438e-07, "loss": 0.0034, "reward": -0.06371062190737575, "reward_after_mean": -0.06371062190737575, "reward_after_std": 0.5247277785092592, "reward_before_mean": 0.32686625607311726, "reward_before_std": 0.4593622125685215, "reward_change_max": 0.0, "reward_change_mean": -0.3905768655240536, "reward_change_min": -0.647777833044529, "reward_change_std": 0.2419142760336399, "reward_std": 0.5247277859598398, "rewards/accuracy_reward": 0.3125000074505806, "rewards/cosine_scaled_reward": 0.014366252347826958, "step": 209 }, { "clip_fraction": 0.0, "completion_length": 3113.250015258789, "epoch": 0.24, "grad_norm": 0.14544087648391724, "kl": 0.008097648620605469, "lambda_div_used": 0.560600072145462, "learning_rate": 7.500858306332172e-07, "loss": 0.009, "reward": -0.21628348529338837, "reward_after_mean": -0.21628348529338837, "reward_after_std": 0.3724058084189892, "reward_before_mean": 0.1631351262331009, "reward_before_std": 0.2722867727279663, "reward_change_max": 0.0, "reward_change_mean": -0.37941864505410194, "reward_change_min": -0.5599995702505112, "reward_change_std": 0.20947875082492828, "reward_std": 0.3724058177322149, "rewards/accuracy_reward": 0.2291666716337204, "rewards/cosine_scaled_reward": -0.0660315528512001, "step": 210 }, { "clip_fraction": 0.0, "completion_length": 2856.437530517578, "epoch": 0.24114285714285713, "grad_norm": 0.18342725932598114, "kl": 0.008159637451171875, "lambda_div_used": 0.5655517652630806, "learning_rate": 7.472670160550848e-07, "loss": 0.0231, "reward": -0.24457528814673424, "reward_after_mean": -0.24457528814673424, "reward_after_std": 0.3880513161420822, "reward_before_mean": 0.11906704865396023, "reward_before_std": 0.29724323377013206, "reward_change_max": 0.0, "reward_change_mean": -0.36364235170185566, "reward_change_min": -0.5528927780687809, "reward_change_std": 0.20731616765260696, "reward_std": 0.3880513347685337, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.08926627226173878, "step": 211 }, { "clip_fraction": 0.0, "completion_length": 3104.979217529297, "epoch": 0.2422857142857143, "grad_norm": 0.1613207757472992, "kl": 0.005316734313964844, "lambda_div_used": 0.5675681903958321, "learning_rate": 7.444385869608921e-07, "loss": 0.0258, "reward": -0.24202940985560417, "reward_after_mean": -0.24202940985560417, "reward_after_std": 0.3931701388210058, "reward_before_mean": 0.12366321124136448, "reward_before_std": 0.3097353223711252, "reward_change_max": 0.0, "reward_change_mean": -0.3656926304101944, "reward_change_min": -0.5545744746923447, "reward_change_std": 0.21149487048387527, "reward_std": 0.39317016303539276, "rewards/accuracy_reward": 0.1875, "rewards/cosine_scaled_reward": -0.06383679807186127, "step": 212 }, { "clip_fraction": 0.0, "completion_length": 3317.0834045410156, "epoch": 0.24342857142857144, "grad_norm": 0.1894315630197525, "kl": 0.015418052673339844, "lambda_div_used": 0.6376521065831184, "learning_rate": 7.416006812042827e-07, "loss": 0.0312, "reward": -0.01567186787724495, "reward_after_mean": -0.01567186787724495, "reward_after_std": 0.6495305839926004, "reward_before_mean": 0.3108948967419565, "reward_before_std": 0.6353255547583103, "reward_change_max": 0.0, "reward_change_mean": -0.32656678929924965, "reward_change_min": -0.5309823006391525, "reward_change_std": 0.21151324920356274, "reward_std": 0.649530591443181, "rewards/accuracy_reward": 0.291666679084301, "rewards/cosine_scaled_reward": 0.01922823116183281, "step": 213 }, { "clip_fraction": 0.0, "completion_length": 3208.4583740234375, "epoch": 0.24457142857142858, "grad_norm": 0.17224323749542236, "kl": 0.009174346923828125, "lambda_div_used": 0.5998109132051468, "learning_rate": 7.387534371007797e-07, "loss": 0.0437, "reward": -0.1456690952181816, "reward_after_mean": -0.1456690952181816, "reward_after_std": 0.5559642724692822, "reward_before_mean": 0.2034774050116539, "reward_before_std": 0.45726070180535316, "reward_change_max": 0.0, "reward_change_mean": -0.34914652816951275, "reward_change_min": -0.5414806716144085, "reward_change_std": 0.20007757656276226, "reward_std": 0.5559643004089594, "rewards/accuracy_reward": 0.2291666679084301, "rewards/cosine_scaled_reward": -0.02568925265222788, "step": 214 }, { "clip_fraction": 0.0, "completion_length": 2805.875030517578, "epoch": 0.24571428571428572, "grad_norm": 0.1782667636871338, "kl": 0.013456344604492188, "lambda_div_used": 0.5679429993033409, "learning_rate": 7.358969934210438e-07, "loss": 0.0162, "reward": -0.37211979553103447, "reward_after_mean": -0.37211979553103447, "reward_after_std": 0.3560109753161669, "reward_before_mean": -0.09492520056664944, "reward_before_std": 0.3112167287617922, "reward_change_max": 0.0, "reward_change_mean": -0.27719458751380444, "reward_change_min": -0.4496432989835739, "reward_change_std": 0.16861796751618385, "reward_std": 0.3560109958052635, "rewards/accuracy_reward": 0.0625, "rewards/cosine_scaled_reward": -0.15742520056664944, "step": 215 }, { "clip_fraction": 0.0, "completion_length": 3008.020835876465, "epoch": 0.24685714285714286, "grad_norm": 0.18901276588439941, "kl": 0.010477066040039062, "lambda_div_used": 0.6216235756874084, "learning_rate": 7.330314893841101e-07, "loss": 0.0147, "reward": 0.00044489139690995216, "reward_after_mean": 0.00044489139690995216, "reward_after_std": 0.644129890948534, "reward_before_mean": 0.3791152648627758, "reward_before_std": 0.5615858174860477, "reward_change_max": 0.0, "reward_change_mean": -0.37867036648094654, "reward_change_min": -0.5948296152055264, "reward_change_std": 0.22754792775958776, "reward_std": 0.6441298983991146, "rewards/accuracy_reward": 0.2916666679084301, "rewards/cosine_scaled_reward": 0.0874485932290554, "step": 216 }, { "clip_fraction": 0.0, "completion_length": 3178.6041870117188, "epoch": 0.248, "grad_norm": 0.16107916831970215, "kl": 0.00598907470703125, "lambda_div_used": 0.5817353799939156, "learning_rate": 7.301570646506027e-07, "loss": 0.0554, "reward": -0.16760935634374619, "reward_after_mean": -0.16760935634374619, "reward_after_std": 0.4673297740519047, "reward_before_mean": 0.20126564521342516, "reward_before_std": 0.37531224731355906, "reward_change_max": 0.0, "reward_change_mean": -0.3688749875873327, "reward_change_min": -0.5384916588664055, "reward_change_std": 0.20844101253896952, "reward_std": 0.4673297815024853, "rewards/accuracy_reward": 0.22916666977107525, "rewards/cosine_scaled_reward": -0.027901039458811283, "step": 217 }, { "clip_fraction": 0.0, "completion_length": 3357.354217529297, "epoch": 0.24914285714285714, "grad_norm": 0.14736585319042206, "kl": 0.009853363037109375, "lambda_div_used": 0.6505353227257729, "learning_rate": 7.27273859315928e-07, "loss": 0.0268, "reward": -0.08380099758505821, "reward_after_mean": -0.08380099758505821, "reward_after_std": 0.7434013560414314, "reward_before_mean": 0.17274162359535694, "reward_before_std": 0.6948446976020932, "reward_change_max": 0.0, "reward_change_mean": -0.25654261931777, "reward_change_min": -0.4430840313434601, "reward_change_std": 0.1646482478827238, "reward_std": 0.7434013988822699, "rewards/accuracy_reward": 0.2083333395421505, "rewards/cosine_scaled_reward": -0.03559170651715249, "step": 218 }, { "clip_fraction": 0.0, "completion_length": 2964.9583435058594, "epoch": 0.2502857142857143, "grad_norm": 0.19797009229660034, "kl": 0.018629074096679688, "lambda_div_used": 0.6038965433835983, "learning_rate": 7.243820139034464e-07, "loss": -0.0078, "reward": -0.1324450857937336, "reward_after_mean": -0.1324450857937336, "reward_after_std": 0.4959396179765463, "reward_before_mean": 0.18938202410936356, "reward_before_std": 0.47722954489290714, "reward_change_max": 0.0, "reward_change_mean": -0.32182709500193596, "reward_change_min": -0.5493551343679428, "reward_change_std": 0.2083377242088318, "reward_std": 0.49593962356448174, "rewards/accuracy_reward": 0.2291666753590107, "rewards/cosine_scaled_reward": -0.039784652180969715, "step": 219 }, { "clip_fraction": 0.0, "completion_length": 3500.8541870117188, "epoch": 0.25142857142857145, "grad_norm": 0.17716443538665771, "kl": 0.016452789306640625, "lambda_div_used": 0.5630562230944633, "learning_rate": 7.214816693576234e-07, "loss": -0.0133, "reward": -0.4373153932392597, "reward_after_mean": -0.4373153932392597, "reward_after_std": 0.34722613357007504, "reward_before_mean": -0.17901553586125374, "reward_before_std": 0.286121791228652, "reward_change_max": 0.0, "reward_change_mean": -0.2582998611032963, "reward_change_min": -0.4369717687368393, "reward_change_std": 0.15314210578799248, "reward_std": 0.34722613357007504, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.22068220796063542, "step": 220 }, { "clip_fraction": 0.0, "completion_length": 2709.000045776367, "epoch": 0.25257142857142856, "grad_norm": 0.1773470640182495, "kl": 0.012754440307617188, "lambda_div_used": 0.5798571780323982, "learning_rate": 7.185729670371604e-07, "loss": -0.0264, "reward": 0.01659686677157879, "reward_after_mean": 0.01659686677157879, "reward_after_std": 0.4947008676826954, "reward_before_mean": 0.5005069419275969, "reward_before_std": 0.36035557463765144, "reward_change_max": 0.0, "reward_change_mean": -0.4839100632816553, "reward_change_min": -0.6756055951118469, "reward_change_std": 0.2605790263041854, "reward_std": 0.4947008863091469, "rewards/accuracy_reward": 0.37500000558793545, "rewards/cosine_scaled_reward": 0.12550692819058895, "step": 221 }, { "clip_fraction": 0.0, "completion_length": 2974.5000610351562, "epoch": 0.2537142857142857, "grad_norm": 0.18921436369419098, "kl": 0.00762939453125, "lambda_div_used": 0.5760413631796837, "learning_rate": 7.156560487081051e-07, "loss": -0.0005, "reward": -0.24993354454636574, "reward_after_mean": -0.24993354454636574, "reward_after_std": 0.45489795319736004, "reward_before_mean": 0.09696098603308201, "reward_before_std": 0.3452626708894968, "reward_change_max": 0.0, "reward_change_mean": -0.34689451940357685, "reward_change_min": -0.5357048697769642, "reward_change_std": 0.19578553270548582, "reward_std": 0.45489797182381153, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.0905390102416277, "step": 222 }, { "clip_fraction": 0.0, "completion_length": 2969.7708740234375, "epoch": 0.25485714285714284, "grad_norm": 0.1810009479522705, "kl": 0.004177093505859375, "lambda_div_used": 0.5716647580265999, "learning_rate": 7.127310565369415e-07, "loss": 0.0433, "reward": -0.14571362175047398, "reward_after_mean": -0.14571362175047398, "reward_after_std": 0.43267372995615005, "reward_before_mean": 0.2578808218240738, "reward_before_std": 0.32162559032440186, "reward_change_max": 0.0, "reward_change_mean": -0.4035944528877735, "reward_change_min": -0.5660542473196983, "reward_change_std": 0.21915364917367697, "reward_std": 0.43267373740673065, "rewards/accuracy_reward": 0.25000000558793545, "rewards/cosine_scaled_reward": 0.007880819961428642, "step": 223 }, { "clip_fraction": 0.0, "completion_length": 3427.7500610351562, "epoch": 0.256, "grad_norm": 0.15454307198524475, "kl": 0.008048057556152344, "lambda_div_used": 0.6068888455629349, "learning_rate": 7.097981330836616e-07, "loss": 0.0385, "reward": -0.21046096831560135, "reward_after_mean": -0.21046096831560135, "reward_after_std": 0.5117912273854017, "reward_before_mean": 0.07308663241565228, "reward_before_std": 0.49874764028936625, "reward_change_max": 0.0, "reward_change_mean": -0.28354761004447937, "reward_change_min": -0.5026295185089111, "reward_change_std": 0.19624011311680079, "reward_std": 0.5117912366986275, "rewards/accuracy_reward": 0.1875000037252903, "rewards/cosine_scaled_reward": -0.1144133610650897, "step": 224 }, { "clip_fraction": 0.0, "completion_length": 3103.520854949951, "epoch": 0.2571428571428571, "grad_norm": 0.18765972554683685, "kl": 0.00830841064453125, "lambda_div_used": 0.6211231797933578, "learning_rate": 7.068574212948169e-07, "loss": 0.0323, "reward": -0.20425227656960487, "reward_after_mean": -0.20425227656960487, "reward_after_std": 0.602248303592205, "reward_before_mean": 0.05286291125230491, "reward_before_std": 0.5583379119634628, "reward_change_max": 0.0, "reward_change_mean": -0.25711517967283726, "reward_change_min": -0.4415777586400509, "reward_change_std": 0.16301549319177866, "reward_std": 0.6022483333945274, "rewards/accuracy_reward": 0.1666666716337204, "rewards/cosine_scaled_reward": -0.11380377411842346, "step": 225 }, { "clip_fraction": 0.0, "completion_length": 3021.479217529297, "epoch": 0.2582857142857143, "grad_norm": 0.15972644090652466, "kl": 0.00731658935546875, "lambda_div_used": 0.5977912843227386, "learning_rate": 7.039090644965509e-07, "loss": 0.0395, "reward": 0.04805717710405588, "reward_after_mean": 0.04805717710405588, "reward_after_std": 0.5278183836489916, "reward_before_mean": 0.4977601831778884, "reward_before_std": 0.4475108031183481, "reward_change_max": 0.0, "reward_change_mean": -0.449703024700284, "reward_change_min": -0.6496801376342773, "reward_change_std": 0.25490943994373083, "reward_std": 0.5278183855116367, "rewards/accuracy_reward": 0.35416666977107525, "rewards/cosine_scaled_reward": 0.14359352039173245, "step": 226 }, { "clip_fraction": 0.0, "completion_length": 3277.5833740234375, "epoch": 0.25942857142857145, "grad_norm": 0.17193704843521118, "kl": 0.017910003662109375, "lambda_div_used": 0.6134023666381836, "learning_rate": 7.009532063876148e-07, "loss": 0.0324, "reward": -0.16794120147824287, "reward_after_mean": -0.16794120147824287, "reward_after_std": 0.5566294267773628, "reward_before_mean": 0.12886080238968134, "reward_before_std": 0.5176326408982277, "reward_change_max": 0.0, "reward_change_mean": -0.29680199548602104, "reward_change_min": -0.48901595920324326, "reward_change_std": 0.184969712048769, "reward_std": 0.5566294398158789, "rewards/accuracy_reward": 0.1875000074505806, "rewards/cosine_scaled_reward": -0.05863920971751213, "step": 227 }, { "clip_fraction": 0.0, "completion_length": 2706.0625076293945, "epoch": 0.26057142857142856, "grad_norm": 0.1777934730052948, "kl": 0.007671356201171875, "lambda_div_used": 0.541731595993042, "learning_rate": 6.979899910323624e-07, "loss": -0.0015, "reward": -0.034288227558135986, "reward_after_mean": -0.034288227558135986, "reward_after_std": 0.3879536911845207, "reward_before_mean": 0.5278026601299644, "reward_before_std": 0.1832903753966093, "reward_change_max": 0.0, "reward_change_mean": -0.5620909035205841, "reward_change_min": -0.764796394854784, "reward_change_std": 0.2873586770147085, "reward_std": 0.3879537060856819, "rewards/accuracy_reward": 0.375, "rewards/cosine_scaled_reward": 0.15280264895409346, "step": 228 }, { "clip_fraction": 0.0, "completion_length": 3471.1458740234375, "epoch": 0.26171428571428573, "grad_norm": 0.1566132754087448, "kl": 0.00818634033203125, "lambda_div_used": 0.5828734710812569, "learning_rate": 6.950195628537299e-07, "loss": 0.0136, "reward": -0.1502060666680336, "reward_after_mean": -0.1502060666680336, "reward_after_std": 0.48206656239926815, "reward_before_mean": 0.24329727701842785, "reward_before_std": 0.3832761161029339, "reward_change_max": 0.0, "reward_change_mean": -0.3935033492743969, "reward_change_min": -0.6296183057129383, "reward_change_std": 0.2353786751627922, "reward_std": 0.4820665866136551, "rewards/accuracy_reward": 0.2291666679084301, "rewards/cosine_scaled_reward": 0.014130610972642899, "step": 229 }, { "clip_fraction": 0.0, "completion_length": 3499.3958435058594, "epoch": 0.26285714285714284, "grad_norm": 0.1700013428926468, "kl": 0.011203765869140625, "lambda_div_used": 0.5797276347875595, "learning_rate": 6.920420666261961e-07, "loss": 0.0136, "reward": -0.3869019029662013, "reward_after_mean": -0.3869019029662013, "reward_after_std": 0.42001777328550816, "reward_before_mean": -0.1338593065738678, "reward_before_std": 0.36143001914024353, "reward_change_max": 0.0, "reward_change_mean": -0.25304259546101093, "reward_change_min": -0.4292117692530155, "reward_change_std": 0.15267042815685272, "reward_std": 0.42001779563724995, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.19635930960066617, "step": 230 }, { "clip_fraction": 0.0, "completion_length": 3200.2083587646484, "epoch": 0.264, "grad_norm": 0.15668541193008423, "kl": 0.0078887939453125, "lambda_div_used": 0.5785987973213196, "learning_rate": 6.890576474687263e-07, "loss": -0.0063, "reward": -0.22548606898635626, "reward_after_mean": -0.22548606898635626, "reward_after_std": 0.4650518875569105, "reward_before_mean": 0.13519820477813482, "reward_before_std": 0.3582014227285981, "reward_change_max": 0.0, "reward_change_mean": -0.3606842774897814, "reward_change_min": -0.5585891306400299, "reward_change_std": 0.203251201659441, "reward_std": 0.46505190059542656, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.05230178916826844, "step": 231 }, { "clip_fraction": 0.0, "completion_length": 3385.0625610351562, "epoch": 0.2651428571428571, "grad_norm": 0.1741892248392105, "kl": 0.0068912506103515625, "lambda_div_used": 0.5824435651302338, "learning_rate": 6.860664508377001e-07, "loss": -0.0075, "reward": -0.38946300745010376, "reward_after_mean": -0.38946300745010376, "reward_after_std": 0.46491232328116894, "reward_before_mean": -0.1540529215708375, "reward_before_std": 0.3710027262568474, "reward_change_max": 0.0, "reward_change_mean": -0.2354100737720728, "reward_change_min": -0.3416454568505287, "reward_change_std": 0.12427148967981339, "reward_std": 0.4649123400449753, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.21655292389914393, "step": 232 }, { "clip_fraction": 0.0, "completion_length": 3153.125030517578, "epoch": 0.2662857142857143, "grad_norm": 0.28357580304145813, "kl": 0.012054443359375, "lambda_div_used": 0.6203712001442909, "learning_rate": 6.83068622519821e-07, "loss": -0.0665, "reward": -0.20462158077862114, "reward_after_mean": -0.20462158077862114, "reward_after_std": 0.599049149081111, "reward_before_mean": 0.058251338079571724, "reward_before_std": 0.5569568537175655, "reward_change_max": 0.0, "reward_change_mean": -0.2628729045391083, "reward_change_min": -0.4553750492632389, "reward_change_std": 0.17058883514255285, "reward_std": 0.5990491714328527, "rewards/accuracy_reward": 0.1458333358168602, "rewards/cosine_scaled_reward": -0.08758200844749808, "step": 233 }, { "clip_fraction": 0.0, "completion_length": 2791.208354949951, "epoch": 0.2674285714285714, "grad_norm": 0.27352628111839294, "kl": 0.01476287841796875, "lambda_div_used": 0.5542727708816528, "learning_rate": 6.800643086250121e-07, "loss": -0.0276, "reward": -0.2108817659318447, "reward_after_mean": -0.2108817659318447, "reward_after_std": 0.3654928673058748, "reward_before_mean": 0.18870040588080883, "reward_before_std": 0.24516455177217722, "reward_change_max": 0.0, "reward_change_mean": -0.3995821550488472, "reward_change_min": -0.5611435957252979, "reward_change_std": 0.21703774482011795, "reward_std": 0.36549287289381027, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.019632935523986816, "step": 234 }, { "clip_fraction": 0.0, "completion_length": 2798.979179382324, "epoch": 0.26857142857142857, "grad_norm": 0.22023151814937592, "kl": 0.023456573486328125, "lambda_div_used": 0.5825880691409111, "learning_rate": 6.770536555792944e-07, "loss": -0.0078, "reward": -0.12622501328587532, "reward_after_mean": -0.12622501328587532, "reward_after_std": 0.46746242977678776, "reward_before_mean": 0.2659615594893694, "reward_before_std": 0.3804177166894078, "reward_change_max": 0.0, "reward_change_mean": -0.3921865876764059, "reward_change_min": -0.6178508289158344, "reward_change_std": 0.23146629706025124, "reward_std": 0.46746244095265865, "rewards/accuracy_reward": 0.2500000037252903, "rewards/cosine_scaled_reward": 0.015961548313498497, "step": 235 }, { "clip_fraction": 0.0, "completion_length": 3211.354217529297, "epoch": 0.26971428571428574, "grad_norm": 0.20094582438468933, "kl": 0.01136016845703125, "lambda_div_used": 0.6315048635005951, "learning_rate": 6.740368101176495e-07, "loss": 0.0152, "reward": -0.05857482645660639, "reward_after_mean": -0.05857482645660639, "reward_after_std": 0.5951094105839729, "reward_before_mean": 0.251810047775507, "reward_before_std": 0.6137803476303816, "reward_change_max": 0.0, "reward_change_mean": -0.310384888201952, "reward_change_min": -0.5718008540570736, "reward_change_std": 0.2273661457002163, "reward_std": 0.5951094273477793, "rewards/accuracy_reward": 0.2708333395421505, "rewards/cosine_scaled_reward": -0.01902329153381288, "step": 236 }, { "clip_fraction": 0.0, "completion_length": 3058.4791870117188, "epoch": 0.27085714285714285, "grad_norm": 0.1709720343351364, "kl": 0.008863449096679688, "lambda_div_used": 0.577639676630497, "learning_rate": 6.710139192768694e-07, "loss": -0.0011, "reward": -0.07855486776679754, "reward_after_mean": -0.07855486776679754, "reward_after_std": 0.5230365730822086, "reward_before_mean": 0.3700827583670616, "reward_before_std": 0.35686243791133165, "reward_change_max": 0.0, "reward_change_mean": -0.4486375879496336, "reward_change_min": -0.6550794281065464, "reward_change_std": 0.24376871157437563, "reward_std": 0.5230365786701441, "rewards/accuracy_reward": 0.31250000186264515, "rewards/cosine_scaled_reward": 0.05758272740058601, "step": 237 }, { "clip_fraction": 0.0, "completion_length": 3248.6250915527344, "epoch": 0.272, "grad_norm": 0.15949378907680511, "kl": 0.0077304840087890625, "lambda_div_used": 0.6372218653559685, "learning_rate": 6.679851303883891e-07, "loss": 0.0272, "reward": 0.16114648059010506, "reward_after_mean": 0.16114648059010506, "reward_after_std": 0.7380091100931168, "reward_before_mean": 0.6169693302363157, "reward_before_std": 0.6436794009059668, "reward_change_max": 0.0, "reward_change_mean": -0.4558228775858879, "reward_change_min": -0.8029187992215157, "reward_change_std": 0.29404174722731113, "reward_std": 0.7380091100931168, "rewards/accuracy_reward": 0.4375000037252903, "rewards/cosine_scaled_reward": 0.1794693369884044, "step": 238 }, { "clip_fraction": 0.0, "completion_length": 2663.1458740234375, "epoch": 0.27314285714285713, "grad_norm": 0.17289391160011292, "kl": 0.007811546325683594, "lambda_div_used": 0.5910399630665779, "learning_rate": 6.649505910711058e-07, "loss": 0.022, "reward": 0.09447231702506542, "reward_after_mean": 0.09447231702506542, "reward_after_std": 0.63307074457407, "reward_before_mean": 0.6295119845308363, "reward_before_std": 0.41379319690167904, "reward_change_max": 0.0, "reward_change_mean": -0.5350396893918514, "reward_change_min": -0.7096161358058453, "reward_change_std": 0.27041073329746723, "reward_std": 0.633070757612586, "rewards/accuracy_reward": 0.43750000186264515, "rewards/cosine_scaled_reward": 0.19201197754591703, "step": 239 }, { "clip_fraction": 0.0, "completion_length": 3472.1458435058594, "epoch": 0.2742857142857143, "grad_norm": 0.18612925708293915, "kl": 0.011730194091796875, "lambda_div_used": 0.5807079896330833, "learning_rate": 6.619104492241847e-07, "loss": 0.003, "reward": -0.3629077561199665, "reward_after_mean": -0.3629077561199665, "reward_after_std": 0.4296929147094488, "reward_before_mean": -0.10262976458761841, "reward_before_std": 0.3699446339160204, "reward_change_max": 0.0, "reward_change_mean": -0.26027799397706985, "reward_change_min": -0.4195384867489338, "reward_change_std": 0.1557039711624384, "reward_std": 0.4296929184347391, "rewards/accuracy_reward": 0.08333333395421505, "rewards/cosine_scaled_reward": -0.18596310168504715, "step": 240 }, { "clip_fraction": 0.0, "completion_length": 3398.9583740234375, "epoch": 0.2754285714285714, "grad_norm": 0.1653418093919754, "kl": 0.010814666748046875, "lambda_div_used": 0.5897242873907089, "learning_rate": 6.588648530198504e-07, "loss": 0.0088, "reward": -0.3430132456123829, "reward_after_mean": -0.3430132456123829, "reward_after_std": 0.4537737797945738, "reward_before_mean": -0.0842779849190265, "reward_before_std": 0.4128823932260275, "reward_change_max": 0.0, "reward_change_mean": -0.25873524136841297, "reward_change_min": -0.4353358559310436, "reward_change_std": 0.16091151162981987, "reward_std": 0.4537738021463156, "rewards/accuracy_reward": 0.08333333395421505, "rewards/cosine_scaled_reward": -0.16761133819818497, "step": 241 }, { "clip_fraction": 0.0, "completion_length": 3044.8125610351562, "epoch": 0.2765714285714286, "grad_norm": 0.19714519381523132, "kl": 0.019735336303710938, "lambda_div_used": 0.5795555114746094, "learning_rate": 6.558139508961654e-07, "loss": -0.0061, "reward": -0.27364013250917196, "reward_after_mean": -0.27364013250917196, "reward_after_std": 0.4104337766766548, "reward_before_mean": 0.03865033481270075, "reward_before_std": 0.36070757918059826, "reward_change_max": 0.0, "reward_change_mean": -0.3122904356569052, "reward_change_min": -0.46563888154923916, "reward_change_std": 0.17774815578013659, "reward_std": 0.41043379716575146, "rewards/accuracy_reward": 0.12500000558793545, "rewards/cosine_scaled_reward": -0.08634967915713787, "step": 242 }, { "clip_fraction": 0.0, "completion_length": 3258.1458587646484, "epoch": 0.2777142857142857, "grad_norm": 0.15476880967617035, "kl": 0.0067596435546875, "lambda_div_used": 0.5753106698393822, "learning_rate": 6.527578915497951e-07, "loss": 0.0152, "reward": -0.21071275137364864, "reward_after_mean": -0.21071275137364864, "reward_after_std": 0.4552135579288006, "reward_before_mean": 0.1430409662425518, "reward_before_std": 0.3436728408560157, "reward_change_max": 0.0, "reward_change_mean": -0.35375371761620045, "reward_change_min": -0.5031162314116955, "reward_change_std": 0.19537147507071495, "reward_std": 0.45521355979144573, "rewards/accuracy_reward": 0.22916666977107525, "rewards/cosine_scaled_reward": -0.08612570352852345, "step": 243 }, { "clip_fraction": 0.0, "completion_length": 3212.000045776367, "epoch": 0.27885714285714286, "grad_norm": 0.16707469522953033, "kl": 0.00936126708984375, "lambda_div_used": 0.6140106841921806, "learning_rate": 6.496968239287603e-07, "loss": 0.0228, "reward": -0.11589334160089493, "reward_after_mean": -0.11589334160089493, "reward_after_std": 0.6261366158723831, "reward_before_mean": 0.23009886965155602, "reward_before_std": 0.52211560215801, "reward_change_max": 0.0, "reward_change_mean": -0.34599223732948303, "reward_change_min": -0.5764485970139503, "reward_change_std": 0.20638986490666866, "reward_std": 0.6261366158723831, "rewards/accuracy_reward": 0.2500000037252903, "rewards/cosine_scaled_reward": -0.01990111800841987, "step": 244 }, { "clip_fraction": 0.0, "completion_length": 3569.375030517578, "epoch": 0.28, "grad_norm": 0.1705743670463562, "kl": 0.007068634033203125, "lambda_div_used": 0.589771993458271, "learning_rate": 6.466308972251785e-07, "loss": 0.0015, "reward": -0.304707333445549, "reward_after_mean": -0.304707333445549, "reward_after_std": 0.4436887241899967, "reward_before_mean": -0.030509795993566513, "reward_before_std": 0.4171254951506853, "reward_change_max": 0.0, "reward_change_mean": -0.27419752813875675, "reward_change_min": -0.4922606311738491, "reward_change_std": 0.18110872618854046, "reward_std": 0.44368873350322247, "rewards/accuracy_reward": 0.1041666679084301, "rewards/cosine_scaled_reward": -0.1346764750778675, "step": 245 }, { "clip_fraction": 0.0, "completion_length": 3165.979217529297, "epoch": 0.28114285714285714, "grad_norm": 0.19632026553153992, "kl": 0.00897216796875, "lambda_div_used": 0.6164729669690132, "learning_rate": 6.435602608679916e-07, "loss": 0.0433, "reward": -0.13705444638617337, "reward_after_mean": -0.13705444638617337, "reward_after_std": 0.5609918963164091, "reward_before_mean": 0.16809637751430273, "reward_before_std": 0.5379356993362308, "reward_change_max": 0.0, "reward_change_mean": -0.30515082366764545, "reward_change_min": -0.5224172919988632, "reward_change_std": 0.20320402085781097, "reward_std": 0.5609919130802155, "rewards/accuracy_reward": 0.22916667349636555, "rewards/cosine_scaled_reward": -0.06107028108090162, "step": 246 }, { "clip_fraction": 0.0, "completion_length": 3451.0416870117188, "epoch": 0.2822857142857143, "grad_norm": 0.14878161251544952, "kl": 0.009401321411132812, "lambda_div_used": 0.5591785162687302, "learning_rate": 6.404850645156841e-07, "loss": 0.0208, "reward": -0.4424121528863907, "reward_after_mean": -0.4424121528863907, "reward_after_std": 0.3408156093209982, "reward_before_mean": -0.1802948098629713, "reward_before_std": 0.2671406352892518, "reward_change_max": 0.0, "reward_change_mean": -0.2621173541992903, "reward_change_min": -0.4238630682229996, "reward_change_std": 0.14910683780908585, "reward_std": 0.34081561863422394, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.22196148382499814, "step": 247 }, { "clip_fraction": 0.0, "completion_length": 3160.562530517578, "epoch": 0.2834285714285714, "grad_norm": 0.21626359224319458, "kl": 0.012750625610351562, "lambda_div_used": 0.5760691612958908, "learning_rate": 6.374054580489873e-07, "loss": 0.0207, "reward": -0.05435580760240555, "reward_after_mean": -0.05435580760240555, "reward_after_std": 0.42985127680003643, "reward_before_mean": 0.37070880085229874, "reward_before_std": 0.34469397366046906, "reward_change_max": 0.0, "reward_change_mean": -0.42506461776793003, "reward_change_min": -0.6026524193584919, "reward_change_std": 0.2407889124006033, "reward_std": 0.4298512954264879, "rewards/accuracy_reward": 0.3333333432674408, "rewards/cosine_scaled_reward": 0.03737547621130943, "step": 248 }, { "clip_fraction": 0.0, "completion_length": 2737.750015258789, "epoch": 0.2845714285714286, "grad_norm": 0.17708338797092438, "kl": 0.019090652465820312, "lambda_div_used": 0.6020805686712265, "learning_rate": 6.343215915635761e-07, "loss": 0.0337, "reward": -0.16037874668836594, "reward_after_mean": -0.16037874668836594, "reward_after_std": 0.5616535171866417, "reward_before_mean": 0.19216054864227772, "reward_before_std": 0.47129171527922153, "reward_change_max": 0.0, "reward_change_mean": -0.3525393009185791, "reward_change_min": -0.5946657881140709, "reward_change_std": 0.21572226658463478, "reward_std": 0.5616535264998674, "rewards/accuracy_reward": 0.22916666977107525, "rewards/cosine_scaled_reward": -0.03700611503154505, "step": 249 }, { "clip_fraction": 0.0, "completion_length": 2989.062530517578, "epoch": 0.2857142857142857, "grad_norm": 0.2667759358882904, "kl": 0.09940719604492188, "lambda_div_used": 0.5786271691322327, "learning_rate": 6.31233615362752e-07, "loss": -0.0573, "reward": -0.39172297716140747, "reward_after_mean": -0.39172297716140747, "reward_after_std": 0.41864336654543877, "reward_before_mean": -0.13376782648265362, "reward_before_std": 0.3546380493789911, "reward_change_max": 0.0, "reward_change_mean": -0.25795516185462475, "reward_change_min": -0.38851144537329674, "reward_change_std": 0.1441606106236577, "reward_std": 0.41864338889718056, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.17543449578806758, "step": 250 }, { "clip_fraction": 0.0, "completion_length": 2677.7916717529297, "epoch": 0.28685714285714287, "grad_norm": 0.19965477287769318, "kl": 0.1493377685546875, "lambda_div_used": 0.683339886367321, "learning_rate": 6.281416799501187e-07, "loss": 0.0249, "reward": 0.08324014954268932, "reward_after_mean": 0.08324014954268932, "reward_after_std": 0.8679631371051073, "reward_before_mean": 0.3540543760173023, "reward_before_std": 0.8538934905081987, "reward_change_max": 0.0, "reward_change_mean": -0.2708142213523388, "reward_change_min": -0.4624214842915535, "reward_change_std": 0.18897927273064852, "reward_std": 0.8679631762206554, "rewards/accuracy_reward": 0.3125000074505806, "rewards/cosine_scaled_reward": 0.041554370895028114, "step": 251 }, { "clip_fraction": 0.0, "completion_length": 3173.1458740234375, "epoch": 0.288, "grad_norm": 0.15656211972236633, "kl": 0.0067844390869140625, "lambda_div_used": 0.5595279112458229, "learning_rate": 6.25045936022246e-07, "loss": 0.02, "reward": -0.20930815115571022, "reward_after_mean": -0.20930815115571022, "reward_after_std": 0.37987689673900604, "reward_before_mean": 0.1839855257421732, "reward_before_std": 0.27092743292450905, "reward_change_max": 0.0, "reward_change_mean": -0.39329367503523827, "reward_change_min": -0.5715558081865311, "reward_change_std": 0.21761398296803236, "reward_std": 0.3798769172281027, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.024347811937332153, "step": 252 }, { "clip_fraction": 0.0, "completion_length": 2972.062545776367, "epoch": 0.28914285714285715, "grad_norm": 0.2085227519273758, "kl": 0.02642822265625, "lambda_div_used": 0.6030574217438698, "learning_rate": 6.219465344613258e-07, "loss": 0.0227, "reward": -0.07461001724004745, "reward_after_mean": -0.07461001724004745, "reward_after_std": 0.49374684877693653, "reward_before_mean": 0.2677174750715494, "reward_before_std": 0.4761771932244301, "reward_change_max": 0.0, "reward_change_mean": -0.3423274699598551, "reward_change_min": -0.5446602664887905, "reward_change_std": 0.21832661796361208, "reward_std": 0.4937468580901623, "rewards/accuracy_reward": 0.2708333432674408, "rewards/cosine_scaled_reward": -0.003115864470601082, "step": 253 }, { "clip_fraction": 0.0, "completion_length": 3405.166717529297, "epoch": 0.29028571428571426, "grad_norm": 0.1530265063047409, "kl": 0.01422119140625, "lambda_div_used": 0.5760461762547493, "learning_rate": 6.188436263278172e-07, "loss": 0.0098, "reward": -0.1361243724822998, "reward_after_mean": -0.1361243724822998, "reward_after_std": 0.4473066031932831, "reward_before_mean": 0.26129917334765196, "reward_before_std": 0.3445596192032099, "reward_change_max": 0.0, "reward_change_mean": -0.3974235448986292, "reward_change_min": -0.5648763813078403, "reward_change_std": 0.21767208073288202, "reward_std": 0.4473066069185734, "rewards/accuracy_reward": 0.25000000558793545, "rewards/cosine_scaled_reward": 0.011299174278974533, "step": 254 }, { "clip_fraction": 0.0, "completion_length": 3231.895835876465, "epoch": 0.2914285714285714, "grad_norm": 0.18611852824687958, "kl": 0.34682464599609375, "lambda_div_used": 0.5978265181183815, "learning_rate": 6.157373628530852e-07, "loss": -0.0144, "reward": -0.3388112420216203, "reward_after_mean": -0.3388112420216203, "reward_after_std": 0.5049593020230532, "reward_before_mean": -0.09141684509813786, "reward_before_std": 0.44909755419939756, "reward_change_max": 0.0, "reward_change_mean": -0.2473943941295147, "reward_change_min": -0.4151032567024231, "reward_change_std": 0.15113264229148626, "reward_std": 0.5049593169242144, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.1747501757927239, "step": 255 }, { "clip_fraction": 0.0, "completion_length": 3386.8958740234375, "epoch": 0.2925714285714286, "grad_norm": 0.16237960755825043, "kl": 0.01018524169921875, "lambda_div_used": 0.6167814210057259, "learning_rate": 6.126278954320294e-07, "loss": 0.0151, "reward": -0.0863467575982213, "reward_after_mean": -0.0863467575982213, "reward_after_std": 0.6445640474557877, "reward_before_mean": 0.26106337551027536, "reward_before_std": 0.5382425542920828, "reward_change_max": 0.0, "reward_change_mean": -0.3474101033061743, "reward_change_min": -0.560859814286232, "reward_change_std": 0.2055121110752225, "reward_std": 0.6445640549063683, "rewards/accuracy_reward": 0.2500000037252903, "rewards/cosine_scaled_reward": 0.01106335874646902, "step": 256 }, { "clip_fraction": 0.0, "completion_length": 3305.2708435058594, "epoch": 0.2937142857142857, "grad_norm": 0.1646636724472046, "kl": 0.0068607330322265625, "lambda_div_used": 0.5782299116253853, "learning_rate": 6.095153756157051e-07, "loss": 0.0264, "reward": -0.1748832268640399, "reward_after_mean": -0.1748832268640399, "reward_after_std": 0.47286594472825527, "reward_before_mean": 0.20799205917865038, "reward_before_std": 0.35885531129315495, "reward_change_max": 0.0, "reward_change_mean": -0.38287530466914177, "reward_change_min": -0.554767481982708, "reward_change_std": 0.2121903682127595, "reward_std": 0.47286595217883587, "rewards/accuracy_reward": 0.20833333395421505, "rewards/cosine_scaled_reward": -0.0003412757068872452, "step": 257 }, { "clip_fraction": 0.0, "completion_length": 3467.1250610351562, "epoch": 0.2948571428571429, "grad_norm": 0.15291546285152435, "kl": 0.008199691772460938, "lambda_div_used": 0.5829475745558739, "learning_rate": 6.06399955103937e-07, "loss": 0.0104, "reward": -0.3416643152013421, "reward_after_mean": -0.3416643152013421, "reward_after_std": 0.43591882660984993, "reward_before_mean": -0.0855697188526392, "reward_before_std": 0.38364509399980307, "reward_change_max": 0.0, "reward_change_mean": -0.25609459541738033, "reward_change_min": -0.4470171108841896, "reward_change_std": 0.16271632071584463, "reward_std": 0.43591883033514023, "rewards/accuracy_reward": 0.1041666679084301, "rewards/cosine_scaled_reward": -0.18973638117313385, "step": 258 }, { "clip_fraction": 0.0, "completion_length": 3050.1666870117188, "epoch": 0.296, "grad_norm": 0.22019357979297638, "kl": 0.011768341064453125, "lambda_div_used": 0.5497855171561241, "learning_rate": 6.032817857379256e-07, "loss": 0.0487, "reward": -0.19476034492254257, "reward_after_mean": -0.19476034492254257, "reward_after_std": 0.35094006918370724, "reward_before_mean": 0.23470514453947544, "reward_before_std": 0.22153367660939693, "reward_change_max": 0.0, "reward_change_mean": -0.4294654652476311, "reward_change_min": -0.5945648476481438, "reward_change_std": 0.2289661392569542, "reward_std": 0.35094008035957813, "rewards/accuracy_reward": 0.2291666716337204, "rewards/cosine_scaled_reward": 0.00553848035633564, "step": 259 }, { "clip_fraction": 0.0, "completion_length": 2537.166702270508, "epoch": 0.29714285714285715, "grad_norm": 0.1907474249601364, "kl": 0.014476776123046875, "lambda_div_used": 0.6038167998194695, "learning_rate": 6.001610194928464e-07, "loss": 0.0303, "reward": -0.03216049447655678, "reward_after_mean": -0.03216049447655678, "reward_after_std": 0.5511109679937363, "reward_before_mean": 0.36014898400753736, "reward_before_std": 0.4787441371008754, "reward_change_max": 0.0, "reward_change_mean": -0.39230949617922306, "reward_change_min": -0.5928047858178616, "reward_change_std": 0.23264547530561686, "reward_std": 0.5511109977960587, "rewards/accuracy_reward": 0.31250000558793545, "rewards/cosine_scaled_reward": 0.04764897469431162, "step": 260 }, { "clip_fraction": 0.0, "completion_length": 3450.604217529297, "epoch": 0.29828571428571427, "grad_norm": 0.16297635436058044, "kl": 0.01052093505859375, "lambda_div_used": 0.5992849767208099, "learning_rate": 5.97037808470444e-07, "loss": -0.0059, "reward": -0.3307666629552841, "reward_after_mean": -0.3307666629552841, "reward_after_std": 0.5031626373529434, "reward_before_mean": -0.07727379910647869, "reward_before_std": 0.45329435355961323, "reward_change_max": 0.0, "reward_change_mean": -0.253492871299386, "reward_change_min": -0.42634226009249687, "reward_change_std": 0.15504747163504362, "reward_std": 0.5031626559793949, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.16060713538900018, "step": 261 }, { "clip_fraction": 0.0, "completion_length": 3433.229217529297, "epoch": 0.29942857142857143, "grad_norm": 0.18257147073745728, "kl": 0.018009185791015625, "lambda_div_used": 0.5564324855804443, "learning_rate": 5.939123048916173e-07, "loss": -0.0083, "reward": -0.4841278623789549, "reward_after_mean": -0.4841278623789549, "reward_after_std": 0.3220012281090021, "reward_before_mean": -0.23878321517258883, "reward_before_std": 0.25116219464689493, "reward_change_max": 0.0, "reward_change_mean": -0.24534465372562408, "reward_change_min": -0.3737174868583679, "reward_change_std": 0.13506192713975906, "reward_std": 0.3220012355595827, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.2596165416762233, "step": 262 }, { "clip_fraction": 0.0, "completion_length": 3113.0625610351562, "epoch": 0.30057142857142854, "grad_norm": 0.16272763907909393, "kl": 0.009136199951171875, "lambda_div_used": 0.5771678313612938, "learning_rate": 5.907846610890011e-07, "loss": 0.0088, "reward": -0.24799314886331558, "reward_after_mean": -0.24799314886331558, "reward_after_std": 0.46621517091989517, "reward_before_mean": 0.09095576778054237, "reward_before_std": 0.34914629347622395, "reward_change_max": 0.0, "reward_change_mean": -0.338948929682374, "reward_change_min": -0.5142081715166569, "reward_change_std": 0.1877481872215867, "reward_std": 0.46621517837047577, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.09654423873871565, "step": 263 }, { "clip_fraction": 0.0, "completion_length": 3100.7708740234375, "epoch": 0.3017142857142857, "grad_norm": 0.21731005609035492, "kl": 0.0093536376953125, "lambda_div_used": 0.5766218304634094, "learning_rate": 5.87655029499542e-07, "loss": 0.0566, "reward": -0.29125285614281893, "reward_after_mean": -0.29125285614281893, "reward_after_std": 0.3911566995084286, "reward_before_mean": 0.005831630900502205, "reward_before_std": 0.3486539348959923, "reward_change_max": 0.0, "reward_change_mean": -0.2970845103263855, "reward_change_min": -0.477193932980299, "reward_change_std": 0.18105328548699617, "reward_std": 0.3911567162722349, "rewards/accuracy_reward": 0.1458333395421505, "rewards/cosine_scaled_reward": -0.1400016937404871, "step": 264 }, { "clip_fraction": 0.0, "completion_length": 3117.062515258789, "epoch": 0.3028571428571429, "grad_norm": 0.16666416823863983, "kl": 0.025203704833984375, "lambda_div_used": 0.6168075576424599, "learning_rate": 5.845235626570683e-07, "loss": 0.0139, "reward": 0.006734441965818405, "reward_after_mean": 0.006734441965818405, "reward_after_std": 0.5516256373375654, "reward_before_mean": 0.3736702101305127, "reward_before_std": 0.5410676412284374, "reward_change_max": 0.0, "reward_change_mean": -0.3669357914477587, "reward_change_min": -0.6034354716539383, "reward_change_std": 0.2386807994917035, "reward_std": 0.551625644788146, "rewards/accuracy_reward": 0.3125000111758709, "rewards/cosine_scaled_reward": 0.06117022875696421, "step": 265 }, { "clip_fraction": 0.0, "completion_length": 3431.9583435058594, "epoch": 0.304, "grad_norm": 0.1549520641565323, "kl": 0.010206222534179688, "lambda_div_used": 0.5356247127056122, "learning_rate": 5.813904131848564e-07, "loss": 0.0048, "reward": -0.5377327017486095, "reward_after_mean": -0.5377327017486095, "reward_after_std": 0.23331154324114323, "reward_before_mean": -0.27887156791985035, "reward_before_std": 0.15639092586934566, "reward_change_max": 0.0, "reward_change_mean": -0.258861118927598, "reward_change_min": -0.3845798037946224, "reward_change_std": 0.13803834281861782, "reward_std": 0.23331154882907867, "rewards/accuracy_reward": 0.0, "rewards/cosine_scaled_reward": -0.2788715772330761, "step": 266 }, { "clip_fraction": 0.0, "completion_length": 3415.2083740234375, "epoch": 0.30514285714285716, "grad_norm": 0.17676624655723572, "kl": 0.013080596923828125, "lambda_div_used": 0.5541348457336426, "learning_rate": 5.78255733788191e-07, "loss": 0.0157, "reward": -0.3900021128356457, "reward_after_mean": -0.3900021128356457, "reward_after_std": 0.30437581427395344, "reward_before_mean": -0.09085199981927872, "reward_before_std": 0.24212567508220673, "reward_change_max": 0.0, "reward_change_mean": -0.2991500999778509, "reward_change_min": -0.44535675272345543, "reward_change_std": 0.1704810056835413, "reward_std": 0.3043758198618889, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.1741853430867195, "step": 267 }, { "clip_fraction": 0.0, "completion_length": 3414.687530517578, "epoch": 0.3062857142857143, "grad_norm": 0.17216652631759644, "kl": 0.022247314453125, "lambda_div_used": 0.5704960450530052, "learning_rate": 5.751196772469237e-07, "loss": 0.0382, "reward": -0.42387382686138153, "reward_after_mean": -0.42387382686138153, "reward_after_std": 0.40421159006655216, "reward_before_mean": -0.176006312482059, "reward_before_std": 0.31731645576655865, "reward_change_max": 0.0, "reward_change_mean": -0.24786749854683876, "reward_change_min": -0.3702392764389515, "reward_change_std": 0.13308451510965824, "reward_std": 0.4042115919291973, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.21767298271879554, "step": 268 }, { "clip_fraction": 0.0, "completion_length": 3232.937530517578, "epoch": 0.30742857142857144, "grad_norm": 0.18452519178390503, "kl": 0.009534835815429688, "lambda_div_used": 0.5792023688554764, "learning_rate": 5.71982396408026e-07, "loss": -0.0092, "reward": -0.2931342124938965, "reward_after_mean": -0.2931342124938965, "reward_after_std": 0.4025785028934479, "reward_before_mean": 0.0003501623868942261, "reward_before_std": 0.3681932780891657, "reward_change_max": 0.0, "reward_change_mean": -0.2934843897819519, "reward_change_min": -0.46174971014261246, "reward_change_std": 0.18471753410995007, "reward_std": 0.402578504756093, "rewards/accuracy_reward": 0.1458333358168602, "rewards/cosine_scaled_reward": -0.14548316970467567, "step": 269 }, { "clip_fraction": 0.0, "completion_length": 3062.1250610351562, "epoch": 0.30857142857142855, "grad_norm": 0.16864223778247833, "kl": 0.02742767333984375, "lambda_div_used": 0.597836896777153, "learning_rate": 5.688440441781398e-07, "loss": -0.0122, "reward": -0.11427849531173706, "reward_after_mean": -0.11427849531173706, "reward_after_std": 0.5567496884614229, "reward_before_mean": 0.24990684166550636, "reward_before_std": 0.4468883480876684, "reward_change_max": 0.0, "reward_change_mean": -0.3641853481531143, "reward_change_min": -0.5258995499461889, "reward_change_std": 0.20153906755149364, "reward_std": 0.5567497033625841, "rewards/accuracy_reward": 0.2500000037252903, "rewards/cosine_scaled_reward": -9.316392242908478e-05, "step": 270 }, { "clip_fraction": 0.0, "completion_length": 2882.854217529297, "epoch": 0.3097142857142857, "grad_norm": 0.19486023485660553, "kl": 0.020715713500976562, "lambda_div_used": 0.6001501008868217, "learning_rate": 5.657047735161255e-07, "loss": 0.0461, "reward": 0.04601459205150604, "reward_after_mean": 0.04601459205150604, "reward_after_std": 0.5237574260681868, "reward_before_mean": 0.48196411691606045, "reward_before_std": 0.45851174369454384, "reward_change_max": 0.0, "reward_change_mean": -0.43594951555132866, "reward_change_min": -0.6822549439966679, "reward_change_std": 0.261988315731287, "reward_std": 0.5237574446946383, "rewards/accuracy_reward": 0.3541666753590107, "rewards/cosine_scaled_reward": 0.12779744528234005, "step": 271 }, { "clip_fraction": 0.0, "completion_length": 3379.1666870117188, "epoch": 0.31085714285714283, "grad_norm": 0.1931447833776474, "kl": 0.012416839599609375, "lambda_div_used": 0.5938794761896133, "learning_rate": 5.625647374256061e-07, "loss": 0.0342, "reward": -0.21854467503726482, "reward_after_mean": -0.21854467503726482, "reward_after_std": 0.4733448401093483, "reward_before_mean": 0.08582894317805767, "reward_before_std": 0.42615535110235214, "reward_change_max": 0.0, "reward_change_mean": -0.30437361635267735, "reward_change_min": -0.5029579028487206, "reward_change_std": 0.18488380871713161, "reward_std": 0.47334484942257404, "rewards/accuracy_reward": 0.16666667349636555, "rewards/cosine_scaled_reward": -0.08083772682584822, "step": 272 }, { "clip_fraction": 0.0, "completion_length": 3204.291702270508, "epoch": 0.312, "grad_norm": 0.16409613192081451, "kl": 0.011150360107421875, "lambda_div_used": 0.560390017926693, "learning_rate": 5.594240889475106e-07, "loss": 0.0155, "reward": -0.0968216024339199, "reward_after_mean": -0.0968216024339199, "reward_after_std": 0.42012024112045765, "reward_before_mean": 0.37499174661934376, "reward_before_std": 0.2746094688773155, "reward_change_max": 0.0, "reward_change_mean": -0.47181335650384426, "reward_change_min": -0.6687533408403397, "reward_change_std": 0.2579321702942252, "reward_std": 0.4201202467083931, "rewards/accuracy_reward": 0.3125, "rewards/cosine_scaled_reward": 0.062491729855537415, "step": 273 }, { "clip_fraction": 0.0, "completion_length": 2519.1458740234375, "epoch": 0.31314285714285717, "grad_norm": 0.2378290742635727, "kl": 0.023778915405273438, "lambda_div_used": 0.6337137371301651, "learning_rate": 5.562829811526154e-07, "loss": -0.0703, "reward": 0.15142151899635792, "reward_after_mean": 0.15142151899635792, "reward_after_std": 0.6880370192229748, "reward_before_mean": 0.5781062543392181, "reward_before_std": 0.6156002655625343, "reward_change_max": 0.0, "reward_change_mean": -0.4266847111284733, "reward_change_min": -0.6568738631904125, "reward_change_std": 0.2552673788741231, "reward_std": 0.6880370192229748, "rewards/accuracy_reward": 0.416666679084301, "rewards/cosine_scaled_reward": 0.16143955197185278, "step": 274 }, { "clip_fraction": 0.0, "completion_length": 2705.979217529297, "epoch": 0.3142857142857143, "grad_norm": 0.1843709498643875, "kl": 0.009296417236328125, "lambda_div_used": 0.5750530734658241, "learning_rate": 5.531415671340826e-07, "loss": 0.0267, "reward": 0.03983314707875252, "reward_after_mean": 0.03983314707875252, "reward_after_std": 0.48223937302827835, "reward_before_mean": 0.5500675980001688, "reward_before_std": 0.341554444283247, "reward_change_max": 0.0, "reward_change_mean": -0.5102344546467066, "reward_change_min": -0.7475596070289612, "reward_change_std": 0.28318499587476254, "reward_std": 0.482239393517375, "rewards/accuracy_reward": 0.3958333395421505, "rewards/cosine_scaled_reward": 0.15423427242785692, "step": 275 }, { "clip_fraction": 0.0, "completion_length": 3254.1250610351562, "epoch": 0.31542857142857145, "grad_norm": 0.1986372023820877, "kl": 0.0174560546875, "lambda_div_used": 0.6367780193686485, "learning_rate": 5.5e-07, "loss": 0.0844, "reward": -0.04936699476093054, "reward_after_mean": -0.04936699476093054, "reward_after_std": 0.6612095236778259, "reward_before_mean": 0.2549672215245664, "reward_before_std": 0.6301376204937696, "reward_change_max": 0.0, "reward_change_mean": -0.3043342139571905, "reward_change_min": -0.5157744195312262, "reward_change_std": 0.19800178240984678, "reward_std": 0.6612095441669226, "rewards/accuracy_reward": 0.2500000074505806, "rewards/cosine_scaled_reward": 0.004967214073985815, "step": 276 }, { "clip_fraction": 0.0, "completion_length": 2920.5208740234375, "epoch": 0.31657142857142856, "grad_norm": 0.2881617546081543, "kl": 0.04803466796875, "lambda_div_used": 0.5883750841021538, "learning_rate": 5.468584328659172e-07, "loss": 0.0824, "reward": -0.27234407141804695, "reward_after_mean": -0.27234407141804695, "reward_after_std": 0.44231759011745453, "reward_before_mean": 0.02457402553409338, "reward_before_std": 0.41091372910887003, "reward_change_max": 0.0, "reward_change_mean": -0.296918086707592, "reward_change_min": -0.5170913115143776, "reward_change_std": 0.19188128411769867, "reward_std": 0.44231759011745453, "rewards/accuracy_reward": 0.1041666679084301, "rewards/cosine_scaled_reward": -0.07959265168756247, "step": 277 }, { "clip_fraction": 0.0, "completion_length": 2999.0834045410156, "epoch": 0.3177142857142857, "grad_norm": 0.18775339424610138, "kl": 0.019683837890625, "lambda_div_used": 0.5474236458539963, "learning_rate": 5.437170188473847e-07, "loss": -0.007, "reward": -0.19653848372399807, "reward_after_mean": -0.19653848372399807, "reward_after_std": 0.3936791978776455, "reward_before_mean": 0.25775109604001045, "reward_before_std": 0.21199503261595964, "reward_change_max": 0.0, "reward_change_mean": -0.45428960770368576, "reward_change_min": -0.614361185580492, "reward_change_std": 0.23127022199332714, "reward_std": 0.39367920719087124, "rewards/accuracy_reward": 0.27083333395421505, "rewards/cosine_scaled_reward": -0.013082241639494896, "step": 278 }, { "clip_fraction": 0.0, "completion_length": 3527.7291870117188, "epoch": 0.31885714285714284, "grad_norm": 0.16561509668827057, "kl": 0.010906219482421875, "lambda_div_used": 0.5985007211565971, "learning_rate": 5.405759110524894e-07, "loss": 0.0217, "reward": -0.3153968108817935, "reward_after_mean": -0.3153968108817935, "reward_after_std": 0.509617468342185, "reward_before_mean": -0.0747665362432599, "reward_before_std": 0.4533124389126897, "reward_change_max": 0.0, "reward_change_mean": -0.2406302783638239, "reward_change_min": -0.39105040580034256, "reward_change_std": 0.1446638461202383, "reward_std": 0.5096174795180559, "rewards/accuracy_reward": 0.1041666679084301, "rewards/cosine_scaled_reward": -0.17893319390714169, "step": 279 }, { "clip_fraction": 0.0, "completion_length": 2680.3958778381348, "epoch": 0.32, "grad_norm": 0.256011962890625, "kl": 2.499143600463867, "lambda_div_used": 0.6519529968500137, "learning_rate": 5.37435262574394e-07, "loss": 0.0396, "reward": 0.13395775854587555, "reward_after_mean": 0.13395775854587555, "reward_after_std": 0.7445417642593384, "reward_before_mean": 0.5125620178878307, "reward_before_std": 0.7079575285315514, "reward_change_max": 0.0, "reward_change_mean": -0.37860422767698765, "reward_change_min": -0.6811250187456608, "reward_change_std": 0.2551812846213579, "reward_std": 0.7445417791604996, "rewards/accuracy_reward": 0.39583334140479565, "rewards/cosine_scaled_reward": 0.11672866437584162, "step": 280 }, { "clip_fraction": 0.0, "completion_length": 3574.6666870117188, "epoch": 0.3211428571428571, "grad_norm": 0.1631581038236618, "kl": 0.008754730224609375, "lambda_div_used": 0.5374941825866699, "learning_rate": 5.342952264838747e-07, "loss": -0.0018, "reward": -0.5442325919866562, "reward_after_mean": -0.5442325919866562, "reward_after_std": 0.24594896659255028, "reward_before_mean": -0.2960541881620884, "reward_before_std": 0.16465971060097218, "reward_change_max": 0.0, "reward_change_mean": -0.24817839823663235, "reward_change_min": -0.36017610132694244, "reward_change_std": 0.1320802504196763, "reward_std": 0.24594897776842117, "rewards/accuracy_reward": 0.0, "rewards/cosine_scaled_reward": -0.2960541881620884, "step": 281 }, { "clip_fraction": 0.0, "completion_length": 3179.937530517578, "epoch": 0.3222857142857143, "grad_norm": 0.16936318576335907, "kl": 0.00806427001953125, "lambda_div_used": 0.580585889518261, "learning_rate": 5.311559558218603e-07, "loss": 0.0141, "reward": 0.0695243813097477, "reward_after_mean": 0.0695243813097477, "reward_after_std": 0.4989807792007923, "reward_before_mean": 0.569744884967804, "reward_before_std": 0.36955196782946587, "reward_change_max": 0.0, "reward_change_mean": -0.5002204813063145, "reward_change_min": -0.7113350480794907, "reward_change_std": 0.28144364431500435, "reward_std": 0.49898079223930836, "rewards/accuracy_reward": 0.4166666716337204, "rewards/cosine_scaled_reward": 0.15307821333408356, "step": 282 }, { "clip_fraction": 0.0, "completion_length": 2918.7500762939453, "epoch": 0.32342857142857145, "grad_norm": 0.1585846245288849, "kl": 0.0053806304931640625, "lambda_div_used": 0.6112403199076653, "learning_rate": 5.28017603591974e-07, "loss": 0.0206, "reward": 0.028243407607078552, "reward_after_mean": 0.028243407607078552, "reward_after_std": 0.6165322158485651, "reward_before_mean": 0.4410540759563446, "reward_before_std": 0.5124128814786673, "reward_change_max": 0.0, "reward_change_mean": -0.41281068325042725, "reward_change_min": -0.663014605641365, "reward_change_std": 0.24524309299886227, "reward_std": 0.6165322568267584, "rewards/accuracy_reward": 0.37500000186264515, "rewards/cosine_scaled_reward": 0.0660540871322155, "step": 283 }, { "clip_fraction": 0.0, "completion_length": 3074.3958740234375, "epoch": 0.32457142857142857, "grad_norm": 0.18992426991462708, "kl": 0.016565322875976562, "lambda_div_used": 0.6459912061691284, "learning_rate": 5.248803227530763e-07, "loss": 0.0623, "reward": -0.008420897647738457, "reward_after_mean": -0.008420897647738457, "reward_after_std": 0.6762434728443623, "reward_before_mean": 0.2930360157042742, "reward_before_std": 0.6791112162172794, "reward_change_max": 0.0, "reward_change_mean": -0.3014569319784641, "reward_change_min": -0.5479001365602016, "reward_change_std": 0.21576991491019726, "reward_std": 0.6762434802949429, "rewards/accuracy_reward": 0.29166667349636555, "rewards/cosine_scaled_reward": 0.0013693645596504211, "step": 284 }, { "clip_fraction": 0.0, "completion_length": 2737.229232788086, "epoch": 0.32571428571428573, "grad_norm": 0.19293582439422607, "kl": 0.010097503662109375, "lambda_div_used": 0.574202872812748, "learning_rate": 5.21744266211809e-07, "loss": 0.0473, "reward": -0.23757071234285831, "reward_after_mean": -0.23757071234285831, "reward_after_std": 0.4655187092721462, "reward_before_mean": 0.11677747126668692, "reward_before_std": 0.3379285214468837, "reward_change_max": 0.0, "reward_change_mean": -0.3543481882661581, "reward_change_min": -0.5226006433367729, "reward_change_std": 0.1939733298495412, "reward_std": 0.4655187278985977, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.0707225389778614, "step": 285 }, { "clip_fraction": 0.0, "completion_length": 3183.8958740234375, "epoch": 0.32685714285714285, "grad_norm": 0.1632191389799118, "kl": 0.0173187255859375, "lambda_div_used": 0.6192898899316788, "learning_rate": 5.186095868151436e-07, "loss": 0.0718, "reward": 0.06020546704530716, "reward_after_mean": 0.06020546704530716, "reward_after_std": 0.5548403132706881, "reward_before_mean": 0.43267707899212837, "reward_before_std": 0.5548370000906289, "reward_change_max": 0.0, "reward_change_mean": -0.37247160635888577, "reward_change_min": -0.6082621030509472, "reward_change_std": 0.24536191765218973, "reward_std": 0.5548403132706881, "rewards/accuracy_reward": 0.354166679084301, "rewards/cosine_scaled_reward": 0.07851038873195648, "step": 286 }, { "clip_fraction": 0.0, "completion_length": 2415.4791831970215, "epoch": 0.328, "grad_norm": 0.24498048424720764, "kl": 0.016057968139648438, "lambda_div_used": 0.6024399250745773, "learning_rate": 5.154764373429315e-07, "loss": 0.0456, "reward": -0.09390892088413239, "reward_after_mean": -0.09390892088413239, "reward_after_std": 0.4796880055218935, "reward_before_mean": 0.2503378726541996, "reward_before_std": 0.4693264402449131, "reward_change_max": 0.0, "reward_change_mean": -0.3442468084394932, "reward_change_min": -0.5578762851655483, "reward_change_std": 0.2223141547292471, "reward_std": 0.47968802228569984, "rewards/accuracy_reward": 0.2500000074505806, "rewards/cosine_scaled_reward": 0.00033786892890930176, "step": 287 }, { "clip_fraction": 0.0, "completion_length": 3160.7916870117188, "epoch": 0.3291428571428571, "grad_norm": 0.17848019301891327, "kl": 0.0107879638671875, "lambda_div_used": 0.5847964808344841, "learning_rate": 5.123449705004581e-07, "loss": 0.0467, "reward": -0.27395089343190193, "reward_after_mean": -0.27395089343190193, "reward_after_std": 0.43479528464376926, "reward_before_mean": 0.01684660091996193, "reward_before_std": 0.39143977500498295, "reward_change_max": 0.0, "reward_change_mean": -0.2907974924892187, "reward_change_min": -0.49354760721325874, "reward_change_std": 0.1831587804481387, "reward_std": 0.4347953088581562, "rewards/accuracy_reward": 0.1250000037252903, "rewards/cosine_scaled_reward": -0.10815340839326382, "step": 288 }, { "clip_fraction": 0.0, "completion_length": 2605.916732788086, "epoch": 0.3302857142857143, "grad_norm": 0.2497103065252304, "kl": 0.109039306640625, "lambda_div_used": 0.5943808779120445, "learning_rate": 5.09215338910999e-07, "loss": 0.0228, "reward": -0.03777727857232094, "reward_after_mean": -0.03777727857232094, "reward_after_std": 0.5212480314075947, "reward_before_mean": 0.377342127263546, "reward_before_std": 0.4325959300622344, "reward_change_max": 0.0, "reward_change_mean": -0.4151194076985121, "reward_change_min": -0.6117738857865334, "reward_change_std": 0.24004015419632196, "reward_std": 0.5212480407208204, "rewards/accuracy_reward": 0.31250000558793545, "rewards/cosine_scaled_reward": 0.06484212819486856, "step": 289 }, { "clip_fraction": 0.0, "completion_length": 3194.604248046875, "epoch": 0.3314285714285714, "grad_norm": 0.17527900636196136, "kl": 0.011566162109375, "lambda_div_used": 0.6183226481080055, "learning_rate": 5.060876951083828e-07, "loss": 0.0286, "reward": -0.03549004439264536, "reward_after_mean": -0.03549004439264536, "reward_after_std": 0.5489960834383965, "reward_before_mean": 0.3164867013692856, "reward_before_std": 0.5450771003961563, "reward_change_max": 0.0, "reward_change_mean": -0.35197674483060837, "reward_change_min": -0.6018749140202999, "reward_change_std": 0.23509527370333672, "reward_std": 0.5489960946142673, "rewards/accuracy_reward": 0.2916666753590107, "rewards/cosine_scaled_reward": 0.024820025078952312, "step": 290 }, { "clip_fraction": 0.0, "completion_length": 3266.854217529297, "epoch": 0.3325714285714286, "grad_norm": 0.1648419201374054, "kl": 0.01015472412109375, "lambda_div_used": 0.6230099350214005, "learning_rate": 5.02962191529556e-07, "loss": -0.0082, "reward": -0.10974331106990576, "reward_after_mean": -0.10974331106990576, "reward_after_std": 0.6065250895917416, "reward_before_mean": 0.18697290122509003, "reward_before_std": 0.5724183414131403, "reward_change_max": 0.0, "reward_change_mean": -0.29671623185276985, "reward_change_min": -0.4786429814994335, "reward_change_std": 0.18716623075306416, "reward_std": 0.6065251212567091, "rewards/accuracy_reward": 0.1875000037252903, "rewards/cosine_scaled_reward": -0.0005270920228213072, "step": 291 }, { "clip_fraction": 0.0, "completion_length": 3522.8958740234375, "epoch": 0.33371428571428574, "grad_norm": 0.1715107411146164, "kl": 0.01177978515625, "lambda_div_used": 0.5558074489235878, "learning_rate": 4.998389805071536e-07, "loss": -0.01, "reward": -0.49656808003783226, "reward_after_mean": -0.49656808003783226, "reward_after_std": 0.3249639421701431, "reward_before_mean": -0.2516378816217184, "reward_before_std": 0.2494716215878725, "reward_change_max": 0.0, "reward_change_mean": -0.2449302077293396, "reward_change_min": -0.35908787697553635, "reward_change_std": 0.13069025985896587, "reward_std": 0.3249639496207237, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.2724712248891592, "step": 292 }, { "clip_fraction": 0.0, "completion_length": 2899.7708892822266, "epoch": 0.33485714285714285, "grad_norm": 0.19758668541908264, "kl": 0.015338897705078125, "lambda_div_used": 0.5540515184402466, "learning_rate": 4.967182142620745e-07, "loss": -0.0222, "reward": -0.21001267433166504, "reward_after_mean": -0.21001267433166504, "reward_after_std": 0.35447113402187824, "reward_before_mean": 0.19157280400395393, "reward_before_std": 0.24082470778375864, "reward_change_max": 0.0, "reward_change_mean": -0.40158548206090927, "reward_change_min": -0.5729983076453209, "reward_change_std": 0.2163494173437357, "reward_std": 0.35447113774716854, "rewards/accuracy_reward": 0.2291666716337204, "rewards/cosine_scaled_reward": -0.03759385272860527, "step": 293 }, { "clip_fraction": 0.0, "completion_length": 3496.312530517578, "epoch": 0.336, "grad_norm": 0.20455823838710785, "kl": 0.0296783447265625, "lambda_div_used": 0.5840108469128609, "learning_rate": 4.93600044896063e-07, "loss": 0.018, "reward": -0.3889755392447114, "reward_after_mean": -0.3889755392447114, "reward_after_std": 0.44188129529356956, "reward_before_mean": -0.14181185828056186, "reward_before_std": 0.384415827691555, "reward_change_max": 0.0, "reward_change_mean": -0.24716367572546005, "reward_change_min": -0.4170133247971535, "reward_change_std": 0.14797675143927336, "reward_std": 0.4418813120573759, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.20431186631321907, "step": 294 }, { "clip_fraction": 0.0, "completion_length": 3439.4166870117188, "epoch": 0.33714285714285713, "grad_norm": 0.189566969871521, "kl": 0.016056060791015625, "lambda_div_used": 0.5852559059858322, "learning_rate": 4.904846243842949e-07, "loss": 0.0087, "reward": -0.2938136998564005, "reward_after_mean": -0.2938136998564005, "reward_after_std": 0.45360755175352097, "reward_before_mean": -0.015423614531755447, "reward_before_std": 0.3863010657951236, "reward_change_max": 0.0, "reward_change_mean": -0.27839008159935474, "reward_change_min": -0.41821156814694405, "reward_change_std": 0.15764420107007027, "reward_std": 0.45360755547881126, "rewards/accuracy_reward": 0.1458333395421505, "rewards/cosine_scaled_reward": -0.1612569612916559, "step": 295 }, { "clip_fraction": 0.0, "completion_length": 3539.125030517578, "epoch": 0.3382857142857143, "grad_norm": 0.18089349567890167, "kl": 0.0132293701171875, "lambda_div_used": 0.5810112357139587, "learning_rate": 4.873721045679706e-07, "loss": 0.0143, "reward": -0.3194497562944889, "reward_after_mean": -0.3194497562944889, "reward_after_std": 0.4224297907203436, "reward_before_mean": -0.04299282981082797, "reward_before_std": 0.3711502766236663, "reward_change_max": 0.0, "reward_change_mean": -0.2764569278806448, "reward_change_min": -0.4383794739842415, "reward_change_std": 0.1645649690181017, "reward_std": 0.4224297907203436, "rewards/accuracy_reward": 0.10416666977107525, "rewards/cosine_scaled_reward": -0.14715949725359678, "step": 296 }, { "clip_fraction": 0.0, "completion_length": 3577.9375, "epoch": 0.3394285714285714, "grad_norm": 0.15882934629917145, "kl": 0.013080596923828125, "lambda_div_used": 0.5605247542262077, "learning_rate": 4.842626371469149e-07, "loss": 0.0035, "reward": -0.436015235260129, "reward_after_mean": -0.436015235260129, "reward_after_std": 0.33564006723463535, "reward_before_mean": -0.1627045925706625, "reward_before_std": 0.2705713417381048, "reward_change_max": 0.0, "reward_change_mean": -0.27331066131591797, "reward_change_min": -0.41958141326904297, "reward_change_std": 0.15145066753029823, "reward_std": 0.3356400802731514, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.1835379246622324, "step": 297 }, { "clip_fraction": 0.0, "completion_length": 3136.062515258789, "epoch": 0.3405714285714286, "grad_norm": 0.17332614958286285, "kl": 0.02989959716796875, "lambda_div_used": 0.5940105840563774, "learning_rate": 4.811563736721829e-07, "loss": -0.0141, "reward": -0.05567885562777519, "reward_after_mean": -0.05567885562777519, "reward_after_std": 0.5181118883192539, "reward_before_mean": 0.34065933059901, "reward_before_std": 0.4283220637589693, "reward_change_max": 0.0, "reward_change_mean": -0.39633818715810776, "reward_change_min": -0.5947207249701023, "reward_change_std": 0.22829789575189352, "reward_std": 0.518111914396286, "rewards/accuracy_reward": 0.33333334140479565, "rewards/cosine_scaled_reward": 0.007325985934585333, "step": 298 }, { "clip_fraction": 0.0, "completion_length": 3346.0208740234375, "epoch": 0.3417142857142857, "grad_norm": 0.1711503267288208, "kl": 0.014190673828125, "lambda_div_used": 0.6083653792738914, "learning_rate": 4.780534655386743e-07, "loss": -0.0072, "reward": -0.0022530220448970795, "reward_after_mean": -0.0022530220448970795, "reward_after_std": 0.565352650359273, "reward_before_mean": 0.4022216070443392, "reward_before_std": 0.5022369045764208, "reward_change_max": 0.0, "reward_change_mean": -0.40447461791336536, "reward_change_min": -0.6457931846380234, "reward_change_std": 0.25260048173367977, "reward_std": 0.5653526708483696, "rewards/accuracy_reward": 0.3125000037252903, "rewards/cosine_scaled_reward": 0.08972159307450056, "step": 299 }, { "clip_fraction": 0.0, "completion_length": 3497.1041870117188, "epoch": 0.34285714285714286, "grad_norm": 0.18349653482437134, "kl": 0.0160675048828125, "lambda_div_used": 0.5405217781662941, "learning_rate": 4.749540639777539e-07, "loss": 0.0158, "reward": -0.5038483627140522, "reward_after_mean": -0.5038483627140522, "reward_after_std": 0.25324913300573826, "reward_before_mean": -0.23165582492947578, "reward_before_std": 0.1779952710494399, "reward_change_max": 0.0, "reward_change_mean": -0.27219252847135067, "reward_change_min": -0.4151998646557331, "reward_change_std": 0.1485830955207348, "reward_std": 0.25324913673102856, "rewards/accuracy_reward": 0.0, "rewards/cosine_scaled_reward": -0.23165582306683064, "step": 300 }, { "clip_fraction": 0.0, "completion_length": 3201.2083435058594, "epoch": 0.344, "grad_norm": 0.184128075838089, "kl": 0.0831451416015625, "lambda_div_used": 0.565784640610218, "learning_rate": 4.7185832004988133e-07, "loss": -0.019, "reward": -0.4314579926431179, "reward_after_mean": -0.4314579926431179, "reward_after_std": 0.35371571965515614, "reward_before_mean": -0.1735248826444149, "reward_before_std": 0.29886128567159176, "reward_change_max": 0.0, "reward_change_mean": -0.25793311931192875, "reward_change_min": -0.43012256920337677, "reward_change_std": 0.15527141094207764, "reward_std": 0.35371571965515614, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.21519154589623213, "step": 301 }, { "clip_fraction": 0.0, "completion_length": 3079.2083587646484, "epoch": 0.34514285714285714, "grad_norm": 0.20796546339988708, "kl": 0.01305389404296875, "lambda_div_used": 0.5801135525107384, "learning_rate": 4.68766384637248e-07, "loss": 0.0271, "reward": -0.10737666673958302, "reward_after_mean": -0.10737666673958302, "reward_after_std": 0.448083333671093, "reward_before_mean": 0.2999982424080372, "reward_before_std": 0.36425076611340046, "reward_change_max": 0.0, "reward_change_mean": -0.4073748905211687, "reward_change_min": -0.6290830448269844, "reward_change_std": 0.2362628672271967, "reward_std": 0.44808334298431873, "rewards/accuracy_reward": 0.2708333395421505, "rewards/cosine_scaled_reward": 0.02916489541530609, "step": 302 }, { "clip_fraction": 0.0, "completion_length": 3114.833335876465, "epoch": 0.3462857142857143, "grad_norm": 0.23826093971729279, "kl": 0.02286529541015625, "lambda_div_used": 0.5798584669828415, "learning_rate": 4.656784084364238e-07, "loss": 0.0033, "reward": -0.21227291598916054, "reward_after_mean": -0.21227291598916054, "reward_after_std": 0.4710402525961399, "reward_before_mean": 0.14477947913110256, "reward_before_std": 0.36395095102488995, "reward_change_max": 0.0, "reward_change_mean": -0.3570523988455534, "reward_change_min": -0.5227542668581009, "reward_change_std": 0.20003251172602177, "reward_std": 0.47104026935994625, "rewards/accuracy_reward": 0.20833333395421505, "rewards/cosine_scaled_reward": -0.06355386786162853, "step": 303 }, { "clip_fraction": 0.0, "completion_length": 3342.625030517578, "epoch": 0.3474285714285714, "grad_norm": 0.17306886613368988, "kl": 0.016788482666015625, "lambda_div_used": 0.5746707916259766, "learning_rate": 4.6259454195101267e-07, "loss": 0.0141, "reward": -0.3166276030242443, "reward_after_mean": -0.3166276030242443, "reward_after_std": 0.39440060034394264, "reward_before_mean": -0.019498261623084545, "reward_before_std": 0.3357888236641884, "reward_change_max": 0.0, "reward_change_mean": -0.2971293330192566, "reward_change_min": -0.43395608849823475, "reward_change_std": 0.16832395922392607, "reward_std": 0.39440061524510384, "rewards/accuracy_reward": 0.12500000558793545, "rewards/cosine_scaled_reward": -0.14449826814234257, "step": 304 }, { "clip_fraction": 0.0, "completion_length": 3286.8959045410156, "epoch": 0.3485714285714286, "grad_norm": 0.17547574639320374, "kl": 0.013736724853515625, "lambda_div_used": 0.5906132087111473, "learning_rate": 4.59514935484316e-07, "loss": 0.0553, "reward": -0.25375326350331306, "reward_after_mean": -0.25375326350331306, "reward_after_std": 0.45609983056783676, "reward_before_mean": 0.030775231309235096, "reward_before_std": 0.41232565976679325, "reward_change_max": 0.0, "reward_change_mean": -0.28452849946916103, "reward_change_min": -0.47182297706604004, "reward_change_std": 0.17629683483392, "reward_std": 0.45609983429312706, "rewards/accuracy_reward": 0.16666667349636555, "rewards/cosine_scaled_reward": -0.13589146081358194, "step": 305 }, { "clip_fraction": 0.0, "completion_length": 3210.354217529297, "epoch": 0.3497142857142857, "grad_norm": 0.14903277158737183, "kl": 0.016412734985351562, "lambda_div_used": 0.5607122406363487, "learning_rate": 4.5643973913200837e-07, "loss": 0.027, "reward": -0.14958194643259048, "reward_after_mean": -0.14958194643259048, "reward_after_std": 0.41496776044368744, "reward_before_mean": 0.30252479389309883, "reward_before_std": 0.2697985488921404, "reward_change_max": 0.0, "reward_change_mean": -0.45210678689181805, "reward_change_min": -0.631548959761858, "reward_change_std": 0.23824660014361143, "reward_std": 0.41496777161955833, "rewards/accuracy_reward": 0.27083333395421505, "rewards/cosine_scaled_reward": 0.03169145993888378, "step": 306 }, { "clip_fraction": 0.0, "completion_length": 3326.2083740234375, "epoch": 0.35085714285714287, "grad_norm": 0.1726515144109726, "kl": 0.013885498046875, "lambda_div_used": 0.6106322109699249, "learning_rate": 4.5336910277482155e-07, "loss": 0.0189, "reward": -0.14274982269853354, "reward_after_mean": -0.14274982269853354, "reward_after_std": 0.5581750143319368, "reward_before_mean": 0.16429251013323665, "reward_before_std": 0.5057175178080797, "reward_change_max": 0.0, "reward_change_mean": -0.30704234167933464, "reward_change_min": -0.5050631985068321, "reward_change_std": 0.1848609447479248, "reward_std": 0.5581750329583883, "rewards/accuracy_reward": 0.1875000074505806, "rewards/cosine_scaled_reward": -0.023207484744489193, "step": 307 }, { "clip_fraction": 0.0, "completion_length": 3568.8958435058594, "epoch": 0.352, "grad_norm": 0.15133541822433472, "kl": 0.014263153076171875, "lambda_div_used": 0.5615487620234489, "learning_rate": 4.503031760712397e-07, "loss": 0.0056, "reward": -0.3069316027686, "reward_after_mean": -0.3069316027686, "reward_after_std": 0.3924997840076685, "reward_before_mean": 0.03872569836676121, "reward_before_std": 0.2771777305752039, "reward_change_max": 0.0, "reward_change_mean": -0.3456573188304901, "reward_change_min": -0.4992937333881855, "reward_change_std": 0.1825523255392909, "reward_std": 0.3924997914582491, "rewards/accuracy_reward": 0.14583333395421505, "rewards/cosine_scaled_reward": -0.10710763186216354, "step": 308 }, { "clip_fraction": 0.0, "completion_length": 3485.3333740234375, "epoch": 0.35314285714285715, "grad_norm": 0.18769589066505432, "kl": 0.011089324951171875, "lambda_div_used": 0.6053318008780479, "learning_rate": 4.4724210845020494e-07, "loss": 0.0224, "reward": -0.26979981176555157, "reward_after_mean": -0.26979981176555157, "reward_after_std": 0.5260053016245365, "reward_before_mean": -0.003813563846051693, "reward_before_std": 0.4861968895420432, "reward_change_max": 0.0, "reward_change_mean": -0.26598624885082245, "reward_change_min": -0.4652111195027828, "reward_change_std": 0.17113024648278952, "reward_std": 0.5260053239762783, "rewards/accuracy_reward": 0.12500000186264515, "rewards/cosine_scaled_reward": -0.12881357036530972, "step": 309 }, { "clip_fraction": 0.0, "completion_length": 3094.625045776367, "epoch": 0.35428571428571426, "grad_norm": 0.1817137897014618, "kl": 0.02679443359375, "lambda_div_used": 0.6295208409428596, "learning_rate": 4.441860491038345e-07, "loss": 0.0122, "reward": -0.11073115654289722, "reward_after_mean": -0.11073115654289722, "reward_after_std": 0.6254717130213976, "reward_before_mean": 0.17308363784104586, "reward_before_std": 0.6048816340044141, "reward_change_max": 0.0, "reward_change_mean": -0.28381480649113655, "reward_change_min": -0.5523712001740932, "reward_change_std": 0.2037146771326661, "reward_std": 0.6254717204719782, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.03524969611316919, "step": 310 }, { "clip_fraction": 0.0, "completion_length": 2869.0834045410156, "epoch": 0.3554285714285714, "grad_norm": 0.21457630395889282, "kl": 0.0152130126953125, "lambda_div_used": 0.5578886866569519, "learning_rate": 4.4113514698014953e-07, "loss": -0.0158, "reward": -0.024519536644220352, "reward_after_mean": -0.024519536644220352, "reward_after_std": 0.41393170692026615, "reward_before_mean": 0.4845733065158129, "reward_before_std": 0.2603567820042372, "reward_change_max": 0.0, "reward_change_mean": -0.5090928487479687, "reward_change_min": -0.7026615478098392, "reward_change_std": 0.2711935807019472, "reward_std": 0.4139317087829113, "rewards/accuracy_reward": 0.3541666716337204, "rewards/cosine_scaled_reward": 0.13040665164589882, "step": 311 }, { "clip_fraction": 0.0, "completion_length": 2765.562545776367, "epoch": 0.3565714285714286, "grad_norm": 0.20729665458202362, "kl": 0.0209808349609375, "lambda_div_used": 0.5657165125012398, "learning_rate": 4.3808955077581546e-07, "loss": 0.0097, "reward": 0.10073104128241539, "reward_after_mean": 0.10073104128241539, "reward_after_std": 0.4854223467409611, "reward_before_mean": 0.6785034229978919, "reward_before_std": 0.2998122461140156, "reward_change_max": 0.0, "reward_change_mean": -0.57777239382267, "reward_change_min": -0.7974594384431839, "reward_change_std": 0.31127106584608555, "reward_std": 0.48542236164212227, "rewards/accuracy_reward": 0.4583333358168602, "rewards/cosine_scaled_reward": 0.2201700694859028, "step": 312 }, { "clip_fraction": 0.0, "completion_length": 3218.6666870117188, "epoch": 0.3577142857142857, "grad_norm": 0.18780964612960815, "kl": 0.010875701904296875, "lambda_div_used": 0.5933279767632484, "learning_rate": 4.350494089288943e-07, "loss": -0.0071, "reward": -0.14533130079507828, "reward_after_mean": -0.14533130079507828, "reward_after_std": 0.504372613504529, "reward_before_mean": 0.21328058652579784, "reward_before_std": 0.43300341442227364, "reward_change_max": 0.0, "reward_change_mean": -0.35861190035939217, "reward_change_min": -0.5982001163065434, "reward_change_std": 0.22190771531313658, "reward_std": 0.5043726228177547, "rewards/accuracy_reward": 0.2291666679084301, "rewards/cosine_scaled_reward": -0.015886055305600166, "step": 313 }, { "clip_fraction": 0.0, "completion_length": 2964.7708892822266, "epoch": 0.3588571428571429, "grad_norm": 0.18988586962223053, "kl": 0.012943267822265625, "lambda_div_used": 0.5895643085241318, "learning_rate": 4.3201486961161093e-07, "loss": 0.0235, "reward": -0.07744442904368043, "reward_after_mean": -0.07744442904368043, "reward_after_std": 0.5397500693798065, "reward_before_mean": 0.35956220095977187, "reward_before_std": 0.4094001241028309, "reward_change_max": 0.0, "reward_change_mean": -0.437006663531065, "reward_change_min": -0.6618683412671089, "reward_change_std": 0.24613509699702263, "reward_std": 0.5397500954568386, "rewards/accuracy_reward": 0.31250000186264515, "rewards/cosine_scaled_reward": 0.04706223588436842, "step": 314 }, { "clip_fraction": 0.0, "completion_length": 3173.479217529297, "epoch": 0.36, "grad_norm": 0.18155807256698608, "kl": 0.019418716430664062, "lambda_div_used": 0.5797837004065514, "learning_rate": 4.2898608072313045e-07, "loss": 0.0318, "reward": -0.08071204647421837, "reward_after_mean": -0.08071204647421837, "reward_after_std": 0.45116767659783363, "reward_before_mean": 0.3363183196634054, "reward_before_std": 0.36499964259564877, "reward_change_max": 0.0, "reward_change_mean": -0.4170303624123335, "reward_change_min": -0.6137964762747288, "reward_change_std": 0.2414401238784194, "reward_std": 0.45116768404841423, "rewards/accuracy_reward": 0.2916666716337204, "rewards/cosine_scaled_reward": 0.044651638716459274, "step": 315 }, { "clip_fraction": 0.0, "completion_length": 3584.0, "epoch": 0.36114285714285715, "grad_norm": 0.170668825507164, "kl": 0.015987396240234375, "lambda_div_used": 0.5594460368156433, "learning_rate": 4.2596318988235037e-07, "loss": -0.0, "reward": -0.40402063727378845, "reward_after_mean": -0.40402063727378845, "reward_after_std": 0.3250133413821459, "reward_before_mean": -0.12532063107937574, "reward_before_std": 0.2696411171928048, "reward_change_max": 0.0, "reward_change_mean": -0.27870001643896103, "reward_change_min": -0.4435790367424488, "reward_change_std": 0.1647225972265005, "reward_std": 0.3250133562833071, "rewards/accuracy_reward": 0.0625, "rewards/cosine_scaled_reward": -0.18782062735408545, "step": 316 }, { "clip_fraction": 0.0, "completion_length": 3443.6666870117188, "epoch": 0.36228571428571427, "grad_norm": 0.19460462033748627, "kl": 0.020721435546875, "lambda_div_used": 0.5617874413728714, "learning_rate": 4.2294634442070553e-07, "loss": 0.0302, "reward": -0.3823830261826515, "reward_after_mean": -0.3823830261826515, "reward_after_std": 0.3326821830123663, "reward_before_mean": -0.10489731654524803, "reward_before_std": 0.28048003278672695, "reward_change_max": 0.0, "reward_change_mean": -0.277485728263855, "reward_change_min": -0.43441783636808395, "reward_change_std": 0.16287430841475725, "reward_std": 0.3326822016388178, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.18823064118623734, "step": 317 }, { "clip_fraction": 0.0, "completion_length": 3026.291717529297, "epoch": 0.36342857142857143, "grad_norm": 0.20331530272960663, "kl": 0.057476043701171875, "lambda_div_used": 0.6181001886725426, "learning_rate": 4.1993569137498776e-07, "loss": -0.0044, "reward": -0.21007327642291784, "reward_after_mean": -0.21007327642291784, "reward_after_std": 0.5821380130946636, "reward_before_mean": 0.05424858466722071, "reward_before_std": 0.5438281707465649, "reward_change_max": 0.0, "reward_change_mean": -0.26432187110185623, "reward_change_min": -0.45752470567822456, "reward_change_std": 0.17214920185506344, "reward_std": 0.5821380130946636, "rewards/accuracy_reward": 0.1666666716337204, "rewards/cosine_scaled_reward": -0.11241808999329805, "step": 318 }, { "clip_fraction": 0.0, "completion_length": 3223.229179382324, "epoch": 0.36457142857142855, "grad_norm": 0.2087068110704422, "kl": 0.05020904541015625, "lambda_div_used": 0.5796853601932526, "learning_rate": 4.1693137748017915e-07, "loss": 0.0098, "reward": -0.3522973582148552, "reward_after_mean": -0.3522973582148552, "reward_after_std": 0.4197389166802168, "reward_before_mean": -0.07807189458981156, "reward_before_std": 0.3631772082298994, "reward_change_max": 0.0, "reward_change_mean": -0.27422546967864037, "reward_change_min": -0.43825293332338333, "reward_change_std": 0.1623068554326892, "reward_std": 0.419738931581378, "rewards/accuracy_reward": 0.08333333395421505, "rewards/cosine_scaled_reward": -0.16140523739159107, "step": 319 }, { "clip_fraction": 0.0, "completion_length": 2755.6458740234375, "epoch": 0.3657142857142857, "grad_norm": 0.23112016916275024, "kl": 0.029541015625, "lambda_div_used": 0.5646919459104538, "learning_rate": 4.1393354916230005e-07, "loss": -0.0155, "reward": -0.07583468779921532, "reward_after_mean": -0.07583468779921532, "reward_after_std": 0.4389583207666874, "reward_before_mean": 0.3969690017402172, "reward_before_std": 0.2951828883960843, "reward_change_max": 0.0, "reward_change_mean": -0.4728037156164646, "reward_change_min": -0.6740865260362625, "reward_change_std": 0.25849280785769224, "reward_std": 0.438958328217268, "rewards/accuracy_reward": 0.3125, "rewards/cosine_scaled_reward": 0.08446897566318512, "step": 320 }, { "clip_fraction": 0.0, "completion_length": 2911.4583892822266, "epoch": 0.3668571428571429, "grad_norm": 0.20181354880332947, "kl": 0.01593017578125, "lambda_div_used": 0.6250731199979782, "learning_rate": 4.1094235253127374e-07, "loss": 0.0487, "reward": -0.014877986162900925, "reward_after_mean": -0.014877986162900925, "reward_after_std": 0.6551479324698448, "reward_before_mean": 0.35487420205026865, "reward_before_std": 0.5803849026560783, "reward_change_max": 0.0, "reward_change_mean": -0.3697521910071373, "reward_change_min": -0.640447624027729, "reward_change_std": 0.23395013436675072, "reward_std": 0.6551479361951351, "rewards/accuracy_reward": 0.29166666977107525, "rewards/cosine_scaled_reward": 0.06320751737803221, "step": 321 }, { "clip_fraction": 0.0, "completion_length": 3304.9583435058594, "epoch": 0.368, "grad_norm": 0.20107248425483704, "kl": 0.05983734130859375, "lambda_div_used": 0.5642069727182388, "learning_rate": 4.079579333738039e-07, "loss": 0.0213, "reward": -0.42735716700553894, "reward_after_mean": -0.42735716700553894, "reward_after_std": 0.34873667918145657, "reward_before_mean": -0.163466926664114, "reward_before_std": 0.2923591611906886, "reward_change_max": 0.0, "reward_change_mean": -0.2638902571052313, "reward_change_min": -0.456204317510128, "reward_change_std": 0.1584412083029747, "reward_std": 0.348736684769392, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.20513359177857637, "step": 322 }, { "clip_fraction": 0.0, "completion_length": 3370.3333435058594, "epoch": 0.36914285714285716, "grad_norm": 0.20461654663085938, "kl": 0.01804351806640625, "lambda_div_used": 0.5772852674126625, "learning_rate": 4.0498043714627006e-07, "loss": 0.048, "reward": -0.2697896584868431, "reward_after_mean": -0.2697896584868431, "reward_after_std": 0.400691881775856, "reward_before_mean": 0.044144206680357456, "reward_before_std": 0.3511424344033003, "reward_change_max": 0.0, "reward_change_mean": -0.31393386237323284, "reward_change_min": -0.5103002227842808, "reward_change_std": 0.18866186309605837, "reward_std": 0.40069189481437206, "rewards/accuracy_reward": 0.1458333395421505, "rewards/cosine_scaled_reward": -0.1016891272738576, "step": 323 }, { "clip_fraction": 0.0, "completion_length": 3356.8333740234375, "epoch": 0.3702857142857143, "grad_norm": 0.17979218065738678, "kl": 0.0131378173828125, "lambda_div_used": 0.5799746140837669, "learning_rate": 4.020100089676376e-07, "loss": -0.0, "reward": -0.22579098492860794, "reward_after_mean": -0.22579098492860794, "reward_after_std": 0.4098988678306341, "reward_before_mean": 0.10009380429983139, "reward_before_std": 0.36687244195491076, "reward_change_max": 0.0, "reward_change_mean": -0.32588481716811657, "reward_change_min": -0.5136618427932262, "reward_change_std": 0.1988009177148342, "reward_std": 0.409898879006505, "rewards/accuracy_reward": 0.1666666716337204, "rewards/cosine_scaled_reward": -0.06657285615801811, "step": 324 }, { "clip_fraction": 0.0, "completion_length": 3210.8958740234375, "epoch": 0.37142857142857144, "grad_norm": 0.18370310962200165, "kl": 0.01052093505859375, "lambda_div_used": 0.5812312439084053, "learning_rate": 3.9904679361238526e-07, "loss": -0.0037, "reward": -0.16739748790860176, "reward_after_mean": -0.16739748790860176, "reward_after_std": 0.46674649976193905, "reward_before_mean": 0.2029289819765836, "reward_before_std": 0.370645004324615, "reward_change_max": 0.0, "reward_change_mean": -0.37032645009458065, "reward_change_min": -0.5316046439111233, "reward_change_std": 0.20442272536456585, "reward_std": 0.4667465090751648, "rewards/accuracy_reward": 0.22916666977107525, "rewards/cosine_scaled_reward": -0.02623770385980606, "step": 325 }, { "clip_fraction": 0.0, "completion_length": 3141.416717529297, "epoch": 0.37257142857142855, "grad_norm": 0.23767949640750885, "kl": 0.014537811279296875, "lambda_div_used": 0.5810681954026222, "learning_rate": 3.9609093550344907e-07, "loss": 0.0518, "reward": -0.27239683270454407, "reward_after_mean": -0.27239683270454407, "reward_after_std": 0.40865136310458183, "reward_before_mean": 0.021625302731990814, "reward_before_std": 0.3728291252627969, "reward_change_max": 0.0, "reward_change_mean": -0.2940221671015024, "reward_change_min": -0.48224707320332527, "reward_change_std": 0.18434689100831747, "reward_std": 0.40865137055516243, "rewards/accuracy_reward": 0.1458333358168602, "rewards/cosine_scaled_reward": -0.12420802749693394, "step": 326 }, { "clip_fraction": 0.0, "completion_length": 3136.3541870117188, "epoch": 0.3737142857142857, "grad_norm": 0.17646071314811707, "kl": 0.015228271484375, "lambda_div_used": 0.6026249304413795, "learning_rate": 3.931425787051832e-07, "loss": -0.0192, "reward": 0.09492400544695556, "reward_after_mean": 0.09492400544695556, "reward_after_std": 0.5985749568790197, "reward_before_mean": 0.5719914492219687, "reward_before_std": 0.4704133979976177, "reward_change_max": 0.0, "reward_change_mean": -0.4770674332976341, "reward_change_min": -0.7213252000510693, "reward_change_std": 0.2710094004869461, "reward_std": 0.5985749736428261, "rewards/accuracy_reward": 0.39583333767950535, "rewards/cosine_scaled_reward": 0.17615810688585043, "step": 327 }, { "clip_fraction": 0.0, "completion_length": 3569.875, "epoch": 0.37485714285714283, "grad_norm": 0.1749822348356247, "kl": 0.017185211181640625, "lambda_div_used": 0.5603252276778221, "learning_rate": 3.902018669163384e-07, "loss": -0.0006, "reward": -0.46640025824308395, "reward_after_mean": -0.46640025824308395, "reward_after_std": 0.3334358800202608, "reward_before_mean": -0.21112712612375617, "reward_before_std": 0.2689880281686783, "reward_change_max": 0.0, "reward_change_mean": -0.25527313351631165, "reward_change_min": -0.4030168130993843, "reward_change_std": 0.143662940710783, "reward_std": 0.3334358874708414, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.23196046613156796, "step": 328 }, { "clip_fraction": 0.0, "completion_length": 2747.812545776367, "epoch": 0.376, "grad_norm": 0.21350613236427307, "kl": 0.01952362060546875, "lambda_div_used": 0.5969379171729088, "learning_rate": 3.872689434630585e-07, "loss": 0.0126, "reward": -0.03132064244709909, "reward_after_mean": -0.03132064244709909, "reward_after_std": 0.6063755955547094, "reward_before_mean": 0.40482214465737343, "reward_before_std": 0.44410499185323715, "reward_change_max": 0.0, "reward_change_mean": -0.436142785474658, "reward_change_min": -0.642087247222662, "reward_change_std": 0.23629819322377443, "reward_std": 0.606375627219677, "rewards/accuracy_reward": 0.3333333358168602, "rewards/cosine_scaled_reward": 0.07148880581371486, "step": 329 }, { "clip_fraction": 0.0, "completion_length": 3380.8958435058594, "epoch": 0.37714285714285717, "grad_norm": 0.1991942673921585, "kl": 0.02968597412109375, "lambda_div_used": 0.5615686550736427, "learning_rate": 3.843439512918949e-07, "loss": 0.0375, "reward": -0.476071247830987, "reward_after_mean": -0.476071247830987, "reward_after_std": 0.3403089623898268, "reward_before_mean": -0.22932932851836085, "reward_before_std": 0.27610256150364876, "reward_change_max": 0.0, "reward_change_mean": -0.24674192070960999, "reward_change_min": -0.3703949488699436, "reward_change_std": 0.13638246525079012, "reward_std": 0.3403089661151171, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.25016266852617264, "step": 330 }, { "clip_fraction": 0.0, "completion_length": 2926.333366394043, "epoch": 0.3782857142857143, "grad_norm": 0.28231188654899597, "kl": 0.02298736572265625, "lambda_div_used": 0.578059084713459, "learning_rate": 3.8142703296283953e-07, "loss": 0.0108, "reward": -0.422471242636675, "reward_after_mean": -0.422471242636675, "reward_after_std": 0.430594801902771, "reward_before_mean": -0.18571816198527813, "reward_before_std": 0.3537705559283495, "reward_change_max": 0.0, "reward_change_mean": -0.23675307631492615, "reward_change_min": -0.3612544909119606, "reward_change_std": 0.127380833029747, "reward_std": 0.4305948093533516, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.2273848308250308, "step": 331 }, { "clip_fraction": 0.0, "completion_length": 3071.7083435058594, "epoch": 0.37942857142857145, "grad_norm": 0.21763376891613007, "kl": 0.017120361328125, "lambda_div_used": 0.575149692595005, "learning_rate": 3.785183306423767e-07, "loss": 0.0062, "reward": -0.09160500764846802, "reward_after_mean": -0.09160500764846802, "reward_after_std": 0.4283015262335539, "reward_before_mean": 0.31884818710386753, "reward_before_std": 0.342169975861907, "reward_change_max": 0.0, "reward_change_mean": -0.41045320592820644, "reward_change_min": -0.5934225358068943, "reward_change_std": 0.23392990790307522, "reward_std": 0.4283015411347151, "rewards/accuracy_reward": 0.3125000074505806, "rewards/cosine_scaled_reward": 0.006348170340061188, "step": 332 }, { "clip_fraction": 0.0, "completion_length": 3059.541717529297, "epoch": 0.38057142857142856, "grad_norm": 0.18071064352989197, "kl": 0.016651153564453125, "lambda_div_used": 0.6297019347548485, "learning_rate": 3.7561798609655373e-07, "loss": -0.0063, "reward": -0.08677977696061134, "reward_after_mean": -0.08677977696061134, "reward_after_std": 0.6162832248955965, "reward_before_mean": 0.20490631833672523, "reward_before_std": 0.6062644021585584, "reward_change_max": 0.0, "reward_change_mean": -0.29168609343469143, "reward_change_min": -0.5152047127485275, "reward_change_std": 0.20360637735575438, "reward_std": 0.6162832360714674, "rewards/accuracy_reward": 0.22916666977107525, "rewards/cosine_scaled_reward": -0.024260364938527346, "step": 333 }, { "clip_fraction": 0.0, "completion_length": 3520.500030517578, "epoch": 0.38171428571428573, "grad_norm": 0.16930466890335083, "kl": 0.0147857666015625, "lambda_div_used": 0.5564875453710556, "learning_rate": 3.72726140684072e-07, "loss": 0.0266, "reward": -0.4434277294203639, "reward_after_mean": -0.4434277294203639, "reward_after_std": 0.33414794877171516, "reward_before_mean": -0.17001175601035357, "reward_before_std": 0.2533196099102497, "reward_change_max": 0.0, "reward_change_mean": -0.2734159901738167, "reward_change_min": -0.39949506893754005, "reward_change_std": 0.14615525864064693, "reward_std": 0.33414795622229576, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.19084508996456861, "step": 334 }, { "clip_fraction": 0.0, "completion_length": 2810.250045776367, "epoch": 0.38285714285714284, "grad_norm": 0.18033389747142792, "kl": 0.014926910400390625, "lambda_div_used": 0.5941435769200325, "learning_rate": 3.6984293534939737e-07, "loss": -0.0036, "reward": 0.12234364170581102, "reward_after_mean": 0.12234364170581102, "reward_after_std": 0.5938457921147346, "reward_before_mean": 0.6426812438294291, "reward_before_std": 0.43207953218370676, "reward_change_max": 0.0, "reward_change_mean": -0.5203375946730375, "reward_change_min": -0.6927892081439495, "reward_change_std": 0.27726735919713974, "reward_std": 0.5938458014279604, "rewards/accuracy_reward": 0.3958333395421505, "rewards/cosine_scaled_reward": 0.2468478727096226, "step": 335 }, { "clip_fraction": 0.0, "completion_length": 3394.250030517578, "epoch": 0.384, "grad_norm": 0.17570967972278595, "kl": 0.01946258544921875, "lambda_div_used": 0.5519177541136742, "learning_rate": 3.6696851061588994e-07, "loss": 0.0235, "reward": -0.2818078063428402, "reward_after_mean": -0.2818078063428402, "reward_after_std": 0.35368830896914005, "reward_before_mean": 0.09115761425346136, "reward_before_std": 0.2350080544129014, "reward_change_max": 0.0, "reward_change_mean": -0.37296542525291443, "reward_change_min": -0.548617996275425, "reward_change_std": 0.20254318416118622, "reward_std": 0.3536883220076561, "rewards/accuracy_reward": 0.1666666679084301, "rewards/cosine_scaled_reward": -0.07550908159464598, "step": 336 }, { "clip_fraction": 0.0, "completion_length": 3360.5208740234375, "epoch": 0.3851428571428571, "grad_norm": 0.18508310616016388, "kl": 0.02142333984375, "lambda_div_used": 0.6332645937800407, "learning_rate": 3.641030065789562e-07, "loss": 0.0214, "reward": -0.10812661051750183, "reward_after_mean": -0.10812661051750183, "reward_after_std": 0.6361647862941027, "reward_before_mean": 0.18243087455630302, "reward_before_std": 0.6123435627669096, "reward_change_max": 0.0, "reward_change_mean": -0.2905574683099985, "reward_change_min": -0.514589611440897, "reward_change_std": 0.19600253272801638, "reward_std": 0.6361648049205542, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.02590246289037168, "step": 337 }, { "clip_fraction": 0.0, "completion_length": 3167.2708740234375, "epoch": 0.3862857142857143, "grad_norm": 0.1991485059261322, "kl": 0.027698516845703125, "lambda_div_used": 0.6096547320485115, "learning_rate": 3.612465628992203e-07, "loss": -0.0018, "reward": -0.03575246036052704, "reward_after_mean": -0.03575246036052704, "reward_after_std": 0.6171925291419029, "reward_before_mean": 0.3501486387103796, "reward_before_std": 0.5004012733697891, "reward_change_max": 0.0, "reward_change_mean": -0.38590112514793873, "reward_change_min": -0.5207641907036304, "reward_change_std": 0.20642881374806166, "reward_std": 0.6171925440430641, "rewards/accuracy_reward": 0.29166667349636555, "rewards/cosine_scaled_reward": 0.05848197266459465, "step": 338 }, { "clip_fraction": 0.0, "completion_length": 3168.375045776367, "epoch": 0.38742857142857146, "grad_norm": 0.16701221466064453, "kl": 0.1025238037109375, "lambda_div_used": 0.5461755990982056, "learning_rate": 3.5839931879571725e-07, "loss": 0.0042, "reward": -0.3618069514632225, "reward_after_mean": -0.3618069514632225, "reward_after_std": 0.3063509054481983, "reward_before_mean": -0.013819726184010506, "reward_before_std": 0.20292026363313198, "reward_change_max": 0.0, "reward_change_mean": -0.34798722714185715, "reward_change_min": -0.5153892748057842, "reward_change_std": 0.18755114544183016, "reward_std": 0.3063509166240692, "rewards/accuracy_reward": 0.125, "rewards/cosine_scaled_reward": -0.13881973549723625, "step": 339 }, { "clip_fraction": 0.0, "completion_length": 3244.8333435058594, "epoch": 0.38857142857142857, "grad_norm": 0.1795177012681961, "kl": 0.01799774169921875, "lambda_div_used": 0.6154248639941216, "learning_rate": 3.555614130391079e-07, "loss": 0.0492, "reward": -0.09047355595976114, "reward_after_mean": -0.09047355595976114, "reward_after_std": 0.5777424033731222, "reward_before_mean": 0.2311236960813403, "reward_before_std": 0.5331126349046826, "reward_change_max": 0.0, "reward_change_mean": -0.32159726694226265, "reward_change_min": -0.48581085726618767, "reward_change_std": 0.19304219540208578, "reward_std": 0.5777424313127995, "rewards/accuracy_reward": 0.2291666753590107, "rewards/cosine_scaled_reward": 0.001957036554813385, "step": 340 }, { "clip_fraction": 0.0, "completion_length": 3112.9375610351562, "epoch": 0.38971428571428574, "grad_norm": 0.16175386309623718, "kl": 0.018917083740234375, "lambda_div_used": 0.5849655121564865, "learning_rate": 3.5273298394491515e-07, "loss": 0.033, "reward": -0.03686077706515789, "reward_after_mean": -0.03686077706515789, "reward_after_std": 0.541625676676631, "reward_before_mean": 0.4353768154978752, "reward_before_std": 0.3934583105146885, "reward_change_max": 0.0, "reward_change_mean": -0.472237603738904, "reward_change_min": -0.7402864620089531, "reward_change_std": 0.2740623615682125, "reward_std": 0.5416257008910179, "rewards/accuracy_reward": 0.3333333358168602, "rewards/cosine_scaled_reward": 0.10204349458217621, "step": 341 }, { "clip_fraction": 0.0, "completion_length": 3160.375030517578, "epoch": 0.39085714285714285, "grad_norm": 0.2217455506324768, "kl": 0.06814193725585938, "lambda_div_used": 0.6448021978139877, "learning_rate": 3.4991416936678276e-07, "loss": 0.0342, "reward": 0.0724026970565319, "reward_after_mean": 0.0724026970565319, "reward_after_std": 0.644262159243226, "reward_before_mean": 0.4257593862712383, "reward_before_std": 0.671525951474905, "reward_change_max": 0.0, "reward_change_mean": -0.3533567041158676, "reward_change_min": -0.6471263207495213, "reward_change_std": 0.25495616532862186, "reward_std": 0.6442621741443872, "rewards/accuracy_reward": 0.354166679084301, "rewards/cosine_scaled_reward": 0.07159272395074368, "step": 342 }, { "clip_fraction": 0.0, "completion_length": 3471.500030517578, "epoch": 0.392, "grad_norm": 0.16222627460956573, "kl": 0.017612457275390625, "lambda_div_used": 0.6022558733820915, "learning_rate": 3.471051066897562e-07, "loss": 0.0221, "reward": -0.19658711925148964, "reward_after_mean": -0.19658711925148964, "reward_after_std": 0.5024736486375332, "reward_before_mean": 0.09291226044297218, "reward_before_std": 0.4669856708496809, "reward_change_max": 0.0, "reward_change_mean": -0.2894993741065264, "reward_change_min": -0.46636413782835007, "reward_change_std": 0.1801355415955186, "reward_std": 0.5024736523628235, "rewards/accuracy_reward": 0.18750000558793545, "rewards/cosine_scaled_reward": -0.09458775259554386, "step": 343 }, { "clip_fraction": 0.0, "completion_length": 2744.479217529297, "epoch": 0.3931428571428571, "grad_norm": 0.16212104260921478, "kl": 0.010890960693359375, "lambda_div_used": 0.5963665023446083, "learning_rate": 3.4430593282358777e-07, "loss": 0.0377, "reward": 0.0931609496474266, "reward_after_mean": 0.0931609496474266, "reward_after_std": 0.5193284433335066, "reward_before_mean": 0.5627899598330259, "reward_before_std": 0.4397473558783531, "reward_change_max": 0.0, "reward_change_mean": -0.4696290194988251, "reward_change_min": -0.7233367376029491, "reward_change_std": 0.27399837225675583, "reward_std": 0.5193284433335066, "rewards/accuracy_reward": 0.4166666716337204, "rewards/cosine_scaled_reward": 0.14612326212227345, "step": 344 }, { "clip_fraction": 0.0, "completion_length": 3405.8333740234375, "epoch": 0.3942857142857143, "grad_norm": 0.20946800708770752, "kl": 0.02022552490234375, "lambda_div_used": 0.5979829281568527, "learning_rate": 3.4151678419606233e-07, "loss": 0.0262, "reward": -0.22470899065956473, "reward_after_mean": -0.22470899065956473, "reward_after_std": 0.4966125153005123, "reward_before_mean": 0.0642069187015295, "reward_before_std": 0.4517026850953698, "reward_change_max": 0.0, "reward_change_mean": -0.2889158893376589, "reward_change_min": -0.47893526405096054, "reward_change_std": 0.18215342983603477, "reward_std": 0.4966125190258026, "rewards/accuracy_reward": 0.14583333395421505, "rewards/cosine_scaled_reward": -0.08162643224932253, "step": 345 }, { "clip_fraction": 0.0, "completion_length": 3305.2500610351562, "epoch": 0.3954285714285714, "grad_norm": 0.1681928187608719, "kl": 0.01726531982421875, "lambda_div_used": 0.5571322664618492, "learning_rate": 3.387377967463493e-07, "loss": 0.0199, "reward": -0.31070056557655334, "reward_after_mean": -0.31070056557655334, "reward_after_std": 0.3715866394340992, "reward_before_mean": 0.04539851937443018, "reward_before_std": 0.25581268686801195, "reward_change_max": 0.0, "reward_change_mean": -0.35609908401966095, "reward_change_min": -0.5095666311681271, "reward_change_std": 0.18707081768661737, "reward_std": 0.3715866431593895, "rewards/accuracy_reward": 0.14583333395421505, "rewards/cosine_scaled_reward": -0.10043482203036547, "step": 346 }, { "clip_fraction": 0.0, "completion_length": 3393.604217529297, "epoch": 0.3965714285714286, "grad_norm": 0.1466827541589737, "kl": 0.011806488037109375, "lambda_div_used": 0.5794221013784409, "learning_rate": 3.359691059183761e-07, "loss": 0.0141, "reward": -0.32797202467918396, "reward_after_mean": -0.32797202467918396, "reward_after_std": 0.40393379144370556, "reward_before_mean": -0.04728060029447079, "reward_before_std": 0.3639182122424245, "reward_change_max": 0.0, "reward_change_mean": -0.2806914187967777, "reward_change_min": -0.48452403768897057, "reward_change_std": 0.177613721229136, "reward_std": 0.40393381007015705, "rewards/accuracy_reward": 0.1041666679084301, "rewards/cosine_scaled_reward": -0.15144727751612663, "step": 347 }, { "clip_fraction": 0.0, "completion_length": 3129.041702270508, "epoch": 0.3977142857142857, "grad_norm": 0.19869239628314972, "kl": 1.1077423095703125, "lambda_div_used": 0.6017066761851311, "learning_rate": 3.3321084665422803e-07, "loss": 0.0276, "reward": -0.2732063550502062, "reward_after_mean": -0.2732063550502062, "reward_after_std": 0.5238662883639336, "reward_before_mean": -0.013186433352530003, "reward_before_std": 0.47198869101703167, "reward_change_max": 0.0, "reward_change_mean": -0.260019913315773, "reward_change_min": -0.45313138142228127, "reward_change_std": 0.16699659638106823, "reward_std": 0.5238662883639336, "rewards/accuracy_reward": 0.12500000186264515, "rewards/cosine_scaled_reward": -0.13818643847480416, "step": 348 }, { "clip_fraction": 0.0, "completion_length": 3329.7708435058594, "epoch": 0.39885714285714285, "grad_norm": 0.17649120092391968, "kl": 0.021068572998046875, "lambda_div_used": 0.5411753505468369, "learning_rate": 3.3046315338757026e-07, "loss": -0.0025, "reward": -0.34042662009596825, "reward_after_mean": -0.34042662009596825, "reward_after_std": 0.29294316098093987, "reward_before_mean": 0.03655768372118473, "reward_before_std": 0.1808468084782362, "reward_change_max": 0.0, "reward_change_mean": -0.37698432989418507, "reward_change_min": -0.5352782905101776, "reward_change_std": 0.20024964958429337, "reward_std": 0.2929431665688753, "rewards/accuracy_reward": 0.125, "rewards/cosine_scaled_reward": -0.08844231441617012, "step": 349 }, { "clip_fraction": 0.0, "completion_length": 3149.2916870117188, "epoch": 0.4, "grad_norm": 0.20012018084526062, "kl": 0.06526947021484375, "lambda_div_used": 0.610220655798912, "learning_rate": 3.2772616003709616e-07, "loss": 0.0369, "reward": -0.2311106538400054, "reward_after_mean": -0.2311106538400054, "reward_after_std": 0.5633708387613297, "reward_before_mean": 0.028151709120720625, "reward_before_std": 0.5049572186544538, "reward_change_max": 0.0, "reward_change_mean": -0.25926235876977444, "reward_change_min": -0.40503259748220444, "reward_change_std": 0.1538803381845355, "reward_std": 0.5633708573877811, "rewards/accuracy_reward": 0.14583333767950535, "rewards/cosine_scaled_reward": -0.11768164113163948, "step": 350 }, { "clip_fraction": 0.0, "completion_length": 3215.062530517578, "epoch": 0.40114285714285713, "grad_norm": 0.1736454963684082, "kl": 0.018604278564453125, "lambda_div_used": 0.6120630353689194, "learning_rate": 3.250000000000001e-07, "loss": 0.0115, "reward": -0.11886338889598846, "reward_after_mean": -0.11886338889598846, "reward_after_std": 0.5669840574264526, "reward_before_mean": 0.19509150739759207, "reward_before_std": 0.5162254478782415, "reward_change_max": 0.0, "reward_change_mean": -0.31395490653812885, "reward_change_min": -0.5001450218260288, "reward_change_std": 0.19280360639095306, "reward_std": 0.5669840592890978, "rewards/accuracy_reward": 0.2083333395421505, "rewards/cosine_scaled_reward": -0.013241829350590706, "step": 351 }, { "clip_fraction": 0.0, "completion_length": 3248.2709350585938, "epoch": 0.4022857142857143, "grad_norm": 0.22111912071704865, "kl": 0.01476287841796875, "lambda_div_used": 0.603014662861824, "learning_rate": 3.222848061454764e-07, "loss": 0.0358, "reward": -0.1326068565249443, "reward_after_mean": -0.1326068565249443, "reward_after_std": 0.48998359218239784, "reward_before_mean": 0.1893708910793066, "reward_before_std": 0.47496388480067253, "reward_change_max": 0.0, "reward_change_mean": -0.3219777401536703, "reward_change_min": -0.5368649587035179, "reward_change_std": 0.21067792922258377, "reward_std": 0.4899835977703333, "rewards/accuracy_reward": 0.2083333395421505, "rewards/cosine_scaled_reward": -0.018962455913424492, "step": 352 }, { "clip_fraction": 0.0, "completion_length": 2886.3334045410156, "epoch": 0.4034285714285714, "grad_norm": 0.1938246190547943, "kl": 0.022388458251953125, "lambda_div_used": 0.5909640118479729, "learning_rate": 3.195807108082429e-07, "loss": 0.0626, "reward": -0.21599614527076483, "reward_after_mean": -0.21599614527076483, "reward_after_std": 0.5302108395844698, "reward_before_mean": 0.1131673906929791, "reward_before_std": 0.4148251572623849, "reward_change_max": 0.0, "reward_change_mean": -0.3291635364294052, "reward_change_min": -0.5086069703102112, "reward_change_std": 0.18236538767814636, "reward_std": 0.5302108637988567, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.09516593092121184, "step": 353 }, { "clip_fraction": 0.0, "completion_length": 2868.104202270508, "epoch": 0.4045714285714286, "grad_norm": 0.19891469180583954, "kl": 0.01898193359375, "lambda_div_used": 0.5724055841565132, "learning_rate": 3.168878457820915e-07, "loss": 0.0179, "reward": -0.03259602561593056, "reward_after_mean": -0.03259602561593056, "reward_after_std": 0.412149578332901, "reward_before_mean": 0.4230206720530987, "reward_before_std": 0.3273854339495301, "reward_change_max": 0.0, "reward_change_mean": -0.455616706982255, "reward_change_min": -0.6391223110258579, "reward_change_std": 0.2547810301184654, "reward_std": 0.41214959882199764, "rewards/accuracy_reward": 0.3333333432674408, "rewards/cosine_scaled_reward": 0.08968737348914146, "step": 354 }, { "clip_fraction": 0.0, "completion_length": 2957.875030517578, "epoch": 0.4057142857142857, "grad_norm": 0.1859559863805771, "kl": 0.01778411865234375, "lambda_div_used": 0.5831267908215523, "learning_rate": 3.142063423134644e-07, "loss": 0.0398, "reward": 0.07190189883112907, "reward_after_mean": 0.07190189883112907, "reward_after_std": 0.49362594820559025, "reward_before_mean": 0.5817297957837582, "reward_before_std": 0.38030726090073586, "reward_change_max": 0.0, "reward_change_mean": -0.5098279379308224, "reward_change_min": -0.736137755215168, "reward_change_std": 0.28942457400262356, "reward_std": 0.4936259649693966, "rewards/accuracy_reward": 0.4166666716337204, "rewards/cosine_scaled_reward": 0.1650631371885538, "step": 355 }, { "clip_fraction": 0.0, "completion_length": 3342.187530517578, "epoch": 0.40685714285714286, "grad_norm": 0.19920627772808075, "kl": 0.0159912109375, "lambda_div_used": 0.6591431573033333, "learning_rate": 3.115363310950578e-07, "loss": 0.0234, "reward": 0.17530879378318787, "reward_after_mean": 0.17530879378318787, "reward_after_std": 0.7574689220637083, "reward_before_mean": 0.5506199598312378, "reward_before_std": 0.7462927037850022, "reward_change_max": 0.0, "reward_change_mean": -0.37531118281185627, "reward_change_min": -0.6638265177607536, "reward_change_std": 0.2594487238675356, "reward_std": 0.7574689388275146, "rewards/accuracy_reward": 0.39583333767950535, "rewards/cosine_scaled_reward": 0.15478662308305502, "step": 356 }, { "clip_fraction": 0.0, "completion_length": 3410.8333740234375, "epoch": 0.408, "grad_norm": 0.18075168132781982, "kl": 0.01677703857421875, "lambda_div_used": 0.5839087069034576, "learning_rate": 3.0887794225945143e-07, "loss": -0.0, "reward": -0.27274375036358833, "reward_after_mean": -0.27274375036358833, "reward_after_std": 0.43362715281546116, "reward_before_mean": 0.021905211731791496, "reward_before_std": 0.3896028930321336, "reward_change_max": 0.0, "reward_change_mean": -0.29464894719421864, "reward_change_min": -0.5043734833598137, "reward_change_std": 0.18624704331159592, "reward_std": 0.4336271844804287, "rewards/accuracy_reward": 0.1250000037252903, "rewards/cosine_scaled_reward": -0.1030947957187891, "step": 357 }, { "clip_fraction": 0.0, "completion_length": 3058.5209045410156, "epoch": 0.40914285714285714, "grad_norm": 0.17896190285682678, "kl": 0.015201568603515625, "lambda_div_used": 0.5992858931422234, "learning_rate": 3.062313053727671e-07, "loss": -0.0262, "reward": 0.08357737958431244, "reward_after_mean": 0.08357737958431244, "reward_after_std": 0.5248461049050093, "reward_before_mean": 0.5405862592160702, "reward_before_std": 0.4560985453426838, "reward_change_max": 0.0, "reward_change_mean": -0.4570088963955641, "reward_change_min": -0.6888911761343479, "reward_change_std": 0.2727729231119156, "reward_std": 0.5248461253941059, "rewards/accuracy_reward": 0.3958333432674408, "rewards/cosine_scaled_reward": 0.14475293457508087, "step": 358 }, { "clip_fraction": 0.0, "completion_length": 3228.500030517578, "epoch": 0.4102857142857143, "grad_norm": 0.17773950099945068, "kl": 0.0243988037109375, "lambda_div_used": 0.5936616808176041, "learning_rate": 3.0359654942835247e-07, "loss": 0.0272, "reward": -0.20648565329611301, "reward_after_mean": -0.20648565329611301, "reward_after_std": 0.5415253769606352, "reward_before_mean": 0.13410826213657856, "reward_before_std": 0.4251509793102741, "reward_change_max": 0.0, "reward_change_mean": -0.3405939098447561, "reward_change_min": -0.48638418316841125, "reward_change_std": 0.18159807473421097, "reward_std": 0.5415253881365061, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.05339175742119551, "step": 359 }, { "clip_fraction": 0.0, "completion_length": 3353.229217529297, "epoch": 0.4114285714285714, "grad_norm": 0.1685146689414978, "kl": 0.016323089599609375, "lambda_div_used": 0.5852215066552162, "learning_rate": 3.0097380284049523e-07, "loss": -0.0127, "reward": -0.07827750593423843, "reward_after_mean": -0.07827750593423843, "reward_after_std": 0.46264146640896797, "reward_before_mean": 0.32931989431381226, "reward_before_std": 0.3890616837888956, "reward_change_max": 0.0, "reward_change_mean": -0.4075974151492119, "reward_change_min": -0.6290528476238251, "reward_change_std": 0.24027447495609522, "reward_std": 0.46264147758483887, "rewards/accuracy_reward": 0.2916666716337204, "rewards/cosine_scaled_reward": 0.03765324130654335, "step": 360 }, { "clip_fraction": 0.0, "completion_length": 3273.4166870117188, "epoch": 0.4125714285714286, "grad_norm": 0.17486868798732758, "kl": 0.01605224609375, "lambda_div_used": 0.6017423197627068, "learning_rate": 2.9836319343816397e-07, "loss": 0.0258, "reward": 0.0011884048581123352, "reward_after_mean": 0.0011884048581123352, "reward_after_std": 0.5392994806170464, "reward_before_mean": 0.41900070011615753, "reward_before_std": 0.4654833022505045, "reward_change_max": 0.0, "reward_change_mean": -0.417812280356884, "reward_change_min": -0.618503101170063, "reward_change_std": 0.24318147636950016, "reward_std": 0.539299488067627, "rewards/accuracy_reward": 0.33333334140479565, "rewards/cosine_scaled_reward": 0.08566735778003931, "step": 361 }, { "clip_fraction": 0.0, "completion_length": 2400.104179382324, "epoch": 0.4137142857142857, "grad_norm": 0.2763527035713196, "kl": 0.02617645263671875, "lambda_div_used": 0.5761683285236359, "learning_rate": 2.9576484845877793e-07, "loss": 0.0729, "reward": -0.06722129508852959, "reward_after_mean": -0.06722129508852959, "reward_after_std": 0.44337410293519497, "reward_before_mean": 0.3675495618954301, "reward_before_std": 0.3499440159648657, "reward_change_max": 0.0, "reward_change_mean": -0.434770867228508, "reward_change_min": -0.6287806890904903, "reward_change_std": 0.24912348575890064, "reward_std": 0.44337411411106586, "rewards/accuracy_reward": 0.2916666716337204, "rewards/cosine_scaled_reward": 0.0758829228579998, "step": 362 }, { "clip_fraction": 0.0, "completion_length": 2630.666717529297, "epoch": 0.41485714285714287, "grad_norm": 0.22403602302074432, "kl": 0.04228973388671875, "lambda_div_used": 0.6052463501691818, "learning_rate": 2.931788945420058e-07, "loss": 0.0504, "reward": -0.10209306981414557, "reward_after_mean": -0.10209306981414557, "reward_after_std": 0.5682412572205067, "reward_before_mean": 0.2561282585375011, "reward_before_std": 0.4855454470962286, "reward_change_max": 0.0, "reward_change_mean": -0.3582213334739208, "reward_change_min": -0.5306066758930683, "reward_change_std": 0.20472939498722553, "reward_std": 0.5682412628084421, "rewards/accuracy_reward": 0.2500000037252903, "rewards/cosine_scaled_reward": 0.006128259003162384, "step": 363 }, { "clip_fraction": 0.0, "completion_length": 3326.937515258789, "epoch": 0.416, "grad_norm": 0.16318903863430023, "kl": 0.02149200439453125, "lambda_div_used": 0.567488469183445, "learning_rate": 2.9060545772359305e-07, "loss": -0.0177, "reward": -0.39752334728837013, "reward_after_mean": -0.39752334728837013, "reward_after_std": 0.3496476709842682, "reward_before_mean": -0.1294215228408575, "reward_before_std": 0.3061747718602419, "reward_change_max": 0.0, "reward_change_mean": -0.2681018318980932, "reward_change_min": -0.44227568432688713, "reward_change_std": 0.16439693607389927, "reward_std": 0.3496476747095585, "rewards/accuracy_reward": 0.0625, "rewards/cosine_scaled_reward": -0.1919215191155672, "step": 364 }, { "clip_fraction": 0.0, "completion_length": 3362.6875610351562, "epoch": 0.41714285714285715, "grad_norm": 0.1784711331129074, "kl": 0.0233154296875, "lambda_div_used": 0.6013159900903702, "learning_rate": 2.8804466342921987e-07, "loss": 0.0207, "reward": -0.26662417873740196, "reward_after_mean": -0.26662417873740196, "reward_after_std": 0.5075900834053755, "reward_before_mean": 0.0036643603816628456, "reward_before_std": 0.465658881701529, "reward_change_max": 0.0, "reward_change_mean": -0.2702885363250971, "reward_change_min": -0.475650642067194, "reward_change_std": 0.17510263249278069, "reward_std": 0.507590102031827, "rewards/accuracy_reward": 0.12500000186264515, "rewards/cosine_scaled_reward": -0.12133564241230488, "step": 365 }, { "clip_fraction": 0.0, "completion_length": 2844.3750762939453, "epoch": 0.41828571428571426, "grad_norm": 0.24511772394180298, "kl": 0.027301788330078125, "lambda_div_used": 0.6240662038326263, "learning_rate": 2.854966364683872e-07, "loss": -0.0512, "reward": 0.2128244184423238, "reward_after_mean": 0.2128244184423238, "reward_after_std": 0.6793836858123541, "reward_before_mean": 0.704135837033391, "reward_before_std": 0.5724440813064575, "reward_change_max": 0.0, "reward_change_mean": -0.4913114458322525, "reward_change_min": -0.7583212815225124, "reward_change_std": 0.2907704198732972, "reward_std": 0.6793837137520313, "rewards/accuracy_reward": 0.45833333767950535, "rewards/cosine_scaled_reward": 0.24580247700214386, "step": 366 }, { "clip_fraction": 0.0, "completion_length": 3170.791748046875, "epoch": 0.41942857142857143, "grad_norm": 0.17575140297412872, "kl": 0.01507568359375, "lambda_div_used": 0.6358509436249733, "learning_rate": 2.829615010283344e-07, "loss": 0.0392, "reward": -0.012937523424625397, "reward_after_mean": -0.012937523424625397, "reward_after_std": 0.6564066410064697, "reward_before_mean": 0.30306646911776625, "reward_before_std": 0.6283687688410282, "reward_change_max": 0.0, "reward_change_mean": -0.3160039931535721, "reward_change_min": -0.526240773499012, "reward_change_std": 0.2031139237806201, "reward_std": 0.6564066577702761, "rewards/accuracy_reward": 0.2708333395421505, "rewards/cosine_scaled_reward": 0.032233125530183315, "step": 367 }, { "clip_fraction": 0.0, "completion_length": 3286.354217529297, "epoch": 0.4205714285714286, "grad_norm": 0.22323228418827057, "kl": 0.01740264892578125, "lambda_div_used": 0.5757176503539085, "learning_rate": 2.8043938066798645e-07, "loss": 0.0368, "reward": -0.2759234309196472, "reward_after_mean": -0.2759234309196472, "reward_after_std": 0.4072870537638664, "reward_before_mean": 0.03852624073624611, "reward_before_std": 0.34614755399525166, "reward_change_max": 0.0, "reward_change_mean": -0.3144496660679579, "reward_change_min": -0.46558523923158646, "reward_change_std": 0.18126972578465939, "reward_std": 0.4072870574891567, "rewards/accuracy_reward": 0.10416666977107525, "rewards/cosine_scaled_reward": -0.06564042856916785, "step": 368 }, { "clip_fraction": 0.0, "completion_length": 3331.4791870117188, "epoch": 0.4217142857142857, "grad_norm": 0.2078586220741272, "kl": 0.024448394775390625, "lambda_div_used": 0.6386851668357849, "learning_rate": 2.7793039831193133e-07, "loss": 0.069, "reward": -0.07261226791888475, "reward_after_mean": -0.07261226791888475, "reward_after_std": 0.6737750265747309, "reward_before_mean": 0.21686414256691933, "reward_before_std": 0.6419388987123966, "reward_change_max": 0.0, "reward_change_mean": -0.28947641141712666, "reward_change_min": -0.46931199356913567, "reward_change_std": 0.1870217639952898, "reward_std": 0.6737750265747309, "rewards/accuracy_reward": 0.20833333767950535, "rewards/cosine_scaled_reward": 0.008530803606845438, "step": 369 }, { "clip_fraction": 0.0, "completion_length": 3365.2916717529297, "epoch": 0.4228571428571429, "grad_norm": 0.17715460062026978, "kl": 0.0180206298828125, "lambda_div_used": 0.5345310568809509, "learning_rate": 2.7543467624442956e-07, "loss": 0.0058, "reward": -0.3411692753434181, "reward_after_mean": -0.3411692753434181, "reward_after_std": 0.28861558996140957, "reward_before_mean": 0.05948095954954624, "reward_before_std": 0.15165453404188156, "reward_change_max": 0.0, "reward_change_mean": -0.40065022744238377, "reward_change_min": -0.5567645654082298, "reward_change_std": 0.2100253812968731, "reward_std": 0.288615595549345, "rewards/accuracy_reward": 0.125, "rewards/cosine_scaled_reward": -0.06551904417574406, "step": 370 }, { "clip_fraction": 0.0, "completion_length": 2519.2291870117188, "epoch": 0.424, "grad_norm": 0.2584461271762848, "kl": 0.033782958984375, "lambda_div_used": 0.5570707470178604, "learning_rate": 2.729523361034538e-07, "loss": 0.044, "reward": -0.038176294416189194, "reward_after_mean": -0.038176294416189194, "reward_after_std": 0.39210474863648415, "reward_before_mean": 0.4650686364620924, "reward_before_std": 0.25540653709322214, "reward_change_max": 0.0, "reward_change_mean": -0.5032449085265398, "reward_change_min": -0.6974077895283699, "reward_change_std": 0.2691148044541478, "reward_std": 0.3921047504991293, "rewards/accuracy_reward": 0.3541666716337204, "rewards/cosine_scaled_reward": 0.11090192757546902, "step": 371 }, { "clip_fraction": 0.0, "completion_length": 3218.979202270508, "epoch": 0.42514285714285716, "grad_norm": 0.16339388489723206, "kl": 0.01953887939453125, "lambda_div_used": 0.5924267843365669, "learning_rate": 2.7048349887476037e-07, "loss": 0.0084, "reward": -0.08437555783893913, "reward_after_mean": -0.08437555783893913, "reward_after_std": 0.5363366901874542, "reward_before_mean": 0.3088786443695426, "reward_before_std": 0.4204003009945154, "reward_change_max": 0.0, "reward_change_mean": -0.3932541720569134, "reward_change_min": -0.5626325234770775, "reward_change_std": 0.21379083301872015, "reward_std": 0.5363367069512606, "rewards/accuracy_reward": 0.2708333395421505, "rewards/cosine_scaled_reward": 0.03804529458284378, "step": 372 }, { "clip_fraction": 0.0, "completion_length": 2564.333366394043, "epoch": 0.42628571428571427, "grad_norm": 0.23124749958515167, "kl": 0.0479736328125, "lambda_div_used": 0.6112604737281799, "learning_rate": 2.6802828488599294e-07, "loss": 0.0761, "reward": -0.22568104229867458, "reward_after_mean": -0.22568104229867458, "reward_after_std": 0.5445360112935305, "reward_before_mean": 0.04894799180328846, "reward_before_std": 0.5200100708752871, "reward_change_max": 0.0, "reward_change_mean": -0.2746290396898985, "reward_change_min": -0.5382625758647919, "reward_change_std": 0.1925811404362321, "reward_std": 0.544536042958498, "rewards/accuracy_reward": 0.1250000037252903, "rewards/cosine_scaled_reward": -0.07605201750993729, "step": 373 }, { "clip_fraction": 0.0, "completion_length": 3052.229202270508, "epoch": 0.42742857142857144, "grad_norm": 0.2122749537229538, "kl": 0.01985931396484375, "lambda_div_used": 0.5861048325896263, "learning_rate": 2.655868138008171e-07, "loss": 0.0071, "reward": -0.09737083874642849, "reward_after_mean": -0.09737083874642849, "reward_after_std": 0.4660142604261637, "reward_before_mean": 0.3094084095209837, "reward_before_std": 0.39544752798974514, "reward_change_max": 0.0, "reward_change_mean": -0.40677923522889614, "reward_change_min": -0.6327800452709198, "reward_change_std": 0.24183003045618534, "reward_std": 0.4660142604261637, "rewards/accuracy_reward": 0.2708333395421505, "rewards/cosine_scaled_reward": 0.038575051352381706, "step": 374 }, { "clip_fraction": 0.0, "completion_length": 3191.25, "epoch": 0.42857142857142855, "grad_norm": 0.2553458511829376, "kl": 0.021575927734375, "lambda_div_used": 0.5968045443296432, "learning_rate": 2.631592046130896e-07, "loss": 0.053, "reward": -0.03919043950736523, "reward_after_mean": -0.03919043950736523, "reward_after_std": 0.5553430318832397, "reward_before_mean": 0.36426131054759026, "reward_before_std": 0.44824791233986616, "reward_change_max": 0.0, "reward_change_mean": -0.4034517649561167, "reward_change_min": -0.5938107855618, "reward_change_std": 0.23507905844599009, "reward_std": 0.5553430542349815, "rewards/accuracy_reward": 0.29166666977107525, "rewards/cosine_scaled_reward": 0.0725946519523859, "step": 375 }, { "clip_fraction": 0.0, "completion_length": 2734.750030517578, "epoch": 0.4297142857142857, "grad_norm": 0.22332127392292023, "kl": 0.025482177734375, "lambda_div_used": 0.5939912125468254, "learning_rate": 2.6074557564105724e-07, "loss": 0.0437, "reward": -0.30326724518090487, "reward_after_mean": -0.30326724518090487, "reward_after_std": 0.4631018824875355, "reward_before_mean": -0.03687694109976292, "reward_before_std": 0.4299264755100012, "reward_change_max": 0.0, "reward_change_mean": -0.2663903124630451, "reward_change_min": -0.471105869859457, "reward_change_std": 0.17796032037585974, "reward_std": 0.46310189366340637, "rewards/accuracy_reward": 0.1875000037252903, "rewards/cosine_scaled_reward": -0.2243769308552146, "step": 376 }, { "clip_fraction": 0.0, "completion_length": 3515.9583740234375, "epoch": 0.4308571428571429, "grad_norm": 0.17636264860630035, "kl": 0.02143096923828125, "lambda_div_used": 0.6144149899482727, "learning_rate": 2.583460445215911e-07, "loss": 0.0138, "reward": -0.20166729297488928, "reward_after_mean": -0.20166729297488928, "reward_after_std": 0.5680250097066164, "reward_before_mean": 0.07150469021871686, "reward_before_std": 0.5254360139369965, "reward_change_max": 0.0, "reward_change_mean": -0.2731720022857189, "reward_change_min": -0.46997616067528725, "reward_change_std": 0.17366825975477695, "reward_std": 0.568025030195713, "rewards/accuracy_reward": 0.1666666716337204, "rewards/cosine_scaled_reward": -0.09516195766627789, "step": 377 }, { "clip_fraction": 0.0, "completion_length": 2912.000011444092, "epoch": 0.432, "grad_norm": 0.27766525745391846, "kl": 0.042694091796875, "lambda_div_used": 0.6006081700325012, "learning_rate": 2.5596072820445254e-07, "loss": 0.0348, "reward": -0.11234386824071407, "reward_after_mean": -0.11234386824071407, "reward_after_std": 0.49495742097496986, "reward_before_mean": 0.2305941702798009, "reward_before_std": 0.4615247752517462, "reward_change_max": 0.0, "reward_change_mean": -0.3429380487650633, "reward_change_min": -0.5275245867669582, "reward_change_std": 0.20726292300969362, "reward_std": 0.4949574302881956, "rewards/accuracy_reward": 0.20833334140479565, "rewards/cosine_scaled_reward": 0.022260839119553566, "step": 378 }, { "clip_fraction": 0.0, "completion_length": 3260.729202270508, "epoch": 0.43314285714285716, "grad_norm": 0.18758617341518402, "kl": 0.03382110595703125, "lambda_div_used": 0.5860039591789246, "learning_rate": 2.5358974294659373e-07, "loss": -0.0237, "reward": -0.27360110729932785, "reward_after_mean": -0.27360110729932785, "reward_after_std": 0.4321169536560774, "reward_before_mean": 0.020669352263212204, "reward_before_std": 0.39550749212503433, "reward_change_max": 0.0, "reward_change_mean": -0.29427043721079826, "reward_change_min": -0.502335362136364, "reward_change_std": 0.18632918037474155, "reward_std": 0.4321169853210449, "rewards/accuracy_reward": 0.1250000037252903, "rewards/cosine_scaled_reward": -0.1043306477367878, "step": 379 }, { "clip_fraction": 0.0, "completion_length": 3021.750045776367, "epoch": 0.4342857142857143, "grad_norm": 0.16766729950904846, "kl": 0.0182037353515625, "lambda_div_used": 0.5926904827356339, "learning_rate": 2.512332043064913e-07, "loss": 0.0344, "reward": -0.12091086152940989, "reward_after_mean": -0.12091086152940989, "reward_after_std": 0.5147394128143787, "reward_before_mean": 0.26931906724348664, "reward_before_std": 0.421288694255054, "reward_change_max": 0.0, "reward_change_mean": -0.3902299255132675, "reward_change_min": -0.620801467448473, "reward_change_std": 0.2289944477379322, "reward_std": 0.5147394314408302, "rewards/accuracy_reward": 0.25000000186264515, "rewards/cosine_scaled_reward": 0.019319066777825356, "step": 380 }, { "clip_fraction": 0.0, "completion_length": 3260.625030517578, "epoch": 0.43542857142857144, "grad_norm": 0.18330129981040955, "kl": 0.02008819580078125, "lambda_div_used": 0.66014763712883, "learning_rate": 2.488912271385139e-07, "loss": -0.0069, "reward": -0.06609210558235645, "reward_after_mean": -0.06609210558235645, "reward_after_std": 0.7662395536899567, "reward_before_mean": 0.1901614892994985, "reward_before_std": 0.7436063382774591, "reward_change_max": 0.0, "reward_change_mean": -0.2562535870820284, "reward_change_min": -0.4897267259657383, "reward_change_std": 0.18116059806197882, "reward_std": 0.766239583492279, "rewards/accuracy_reward": 0.25000000931322575, "rewards/cosine_scaled_reward": -0.05983851058408618, "step": 381 }, { "clip_fraction": 0.0, "completion_length": 3129.625030517578, "epoch": 0.43657142857142855, "grad_norm": 0.23972399532794952, "kl": 0.019500732421875, "lambda_div_used": 0.5837963595986366, "learning_rate": 2.465639255873246e-07, "loss": 0.0536, "reward": -0.3467144723981619, "reward_after_mean": -0.3467144723981619, "reward_after_std": 0.43434189446270466, "reward_before_mean": -0.07917244592681527, "reward_before_std": 0.3838723711669445, "reward_change_max": 0.0, "reward_change_mean": -0.26754201017320156, "reward_change_min": -0.43388141691684723, "reward_change_std": 0.16061161924153566, "reward_std": 0.4343419149518013, "rewards/accuracy_reward": 0.08333333395421505, "rewards/cosine_scaled_reward": -0.16250578872859478, "step": 382 }, { "clip_fraction": 0.0, "completion_length": 3411.7083740234375, "epoch": 0.4377142857142857, "grad_norm": 0.23316802084445953, "kl": 0.0207977294921875, "lambda_div_used": 0.6381125152111053, "learning_rate": 2.4425141308231765e-07, "loss": 0.055, "reward": -0.07424248196184635, "reward_after_mean": -0.07424248196184635, "reward_after_std": 0.639557383954525, "reward_before_mean": 0.20887607336044312, "reward_before_std": 0.6492404751479626, "reward_change_max": 0.0, "reward_change_mean": -0.2831185795366764, "reward_change_min": -0.564945362508297, "reward_change_std": 0.21358319744467735, "reward_std": 0.6395574305206537, "rewards/accuracy_reward": 0.2291666716337204, "rewards/cosine_scaled_reward": -0.020290587097406387, "step": 383 }, { "clip_fraction": 0.0, "completion_length": 2741.9583435058594, "epoch": 0.43885714285714283, "grad_norm": 0.25789573788642883, "kl": 0.07757949829101562, "lambda_div_used": 0.6318844854831696, "learning_rate": 2.4195380233209006e-07, "loss": 0.0677, "reward": 0.19942376017570496, "reward_after_mean": 0.19942376017570496, "reward_after_std": 0.7111247107386589, "reward_before_mean": 0.6753224628046155, "reward_before_std": 0.6163387764245272, "reward_change_max": 0.0, "reward_change_mean": -0.4758987110108137, "reward_change_min": -0.7505817860364914, "reward_change_std": 0.29390372429043055, "reward_std": 0.7111247181892395, "rewards/accuracy_reward": 0.45833333395421505, "rewards/cosine_scaled_reward": 0.21698913536965847, "step": 384 }, { "clip_fraction": 0.0, "completion_length": 3379.4375610351562, "epoch": 0.44, "grad_norm": 0.19184155762195587, "kl": 0.0209503173828125, "lambda_div_used": 0.5597429350018501, "learning_rate": 2.3967120531894857e-07, "loss": 0.0132, "reward": -0.42902201041579247, "reward_after_mean": -0.42902201041579247, "reward_after_std": 0.32078694365918636, "reward_before_mean": -0.15332065522670746, "reward_before_std": 0.26833571307361126, "reward_change_max": 0.0, "reward_change_mean": -0.2757013589143753, "reward_change_min": -0.4565298557281494, "reward_change_std": 0.16292693745344877, "reward_std": 0.3207869604229927, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.19498731568455696, "step": 385 }, { "clip_fraction": 0.0, "completion_length": 3332.979217529297, "epoch": 0.44114285714285717, "grad_norm": 0.17706023156642914, "kl": 0.0265960693359375, "lambda_div_used": 0.5816922634840012, "learning_rate": 2.374037332934512e-07, "loss": 0.0052, "reward": -0.3098838999867439, "reward_after_mean": -0.3098838999867439, "reward_after_std": 0.40603481233119965, "reward_before_mean": -0.028107697144150734, "reward_before_std": 0.37378187756985426, "reward_change_max": 0.0, "reward_change_mean": -0.2817761991173029, "reward_change_min": -0.4757208116352558, "reward_change_std": 0.17866256646811962, "reward_std": 0.40603483095765114, "rewards/accuracy_reward": 0.1250000037252903, "rewards/cosine_scaled_reward": -0.15310769528150558, "step": 386 }, { "clip_fraction": 0.0, "completion_length": 3277.9791870117188, "epoch": 0.4422857142857143, "grad_norm": 0.18167604506015778, "kl": 0.0161285400390625, "lambda_div_used": 0.5914167314767838, "learning_rate": 2.3515149676898552e-07, "loss": 0.009, "reward": -0.34463197365403175, "reward_after_mean": -0.34463197365403175, "reward_after_std": 0.4892254173755646, "reward_before_mean": -0.09299218654632568, "reward_before_std": 0.4171148231253028, "reward_change_max": 0.0, "reward_change_mean": -0.2516398001462221, "reward_change_min": -0.4150817207992077, "reward_change_std": 0.1475861668586731, "reward_std": 0.48922544345259666, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.17632551956921816, "step": 387 }, { "clip_fraction": 0.0, "completion_length": 3205.291717529297, "epoch": 0.44342857142857145, "grad_norm": 0.1990031599998474, "kl": 0.02637481689453125, "lambda_div_used": 0.5404903516173363, "learning_rate": 2.3291460551638237e-07, "loss": 0.0031, "reward": -0.21684178709983826, "reward_after_mean": -0.21684178709983826, "reward_after_std": 0.3337708804756403, "reward_before_mean": 0.2303929552435875, "reward_before_std": 0.17789103090763092, "reward_change_max": 0.0, "reward_change_mean": -0.4472347106784582, "reward_change_min": -0.6312463767826557, "reward_change_std": 0.23187166824936867, "reward_std": 0.3337708879262209, "rewards/accuracy_reward": 0.25, "rewards/cosine_scaled_reward": -0.01960706152021885, "step": 388 }, { "clip_fraction": 0.0, "completion_length": 3192.437530517578, "epoch": 0.44457142857142856, "grad_norm": 0.2047928422689438, "kl": 0.0220489501953125, "lambda_div_used": 0.5579963997006416, "learning_rate": 2.306931685585657e-07, "loss": 0.0229, "reward": -0.21862464770674706, "reward_after_mean": -0.21862464770674706, "reward_after_std": 0.372770469635725, "reward_before_mean": 0.16921952925622463, "reward_before_std": 0.2651206851005554, "reward_change_max": 0.0, "reward_change_mean": -0.38784417882561684, "reward_change_min": -0.545613270252943, "reward_change_std": 0.2151289852336049, "reward_std": 0.3727704808115959, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.039113814011216164, "step": 389 }, { "clip_fraction": 0.0, "completion_length": 3141.7084197998047, "epoch": 0.44571428571428573, "grad_norm": 0.1907465159893036, "kl": 0.01737213134765625, "lambda_div_used": 0.64802136272192, "learning_rate": 2.2848729416523859e-07, "loss": 0.0454, "reward": -0.08123735431581736, "reward_after_mean": -0.08123735431581736, "reward_after_std": 0.697978412732482, "reward_before_mean": 0.18789247795939445, "reward_before_std": 0.6907994262874126, "reward_change_max": 0.0, "reward_change_mean": -0.26912982389330864, "reward_change_min": -0.5454708561301231, "reward_change_std": 0.2026740238070488, "reward_std": 0.6979784555733204, "rewards/accuracy_reward": 0.22916666977107525, "rewards/cosine_scaled_reward": -0.04127418214920908, "step": 390 }, { "clip_fraction": 0.0, "completion_length": 3047.7708740234375, "epoch": 0.44685714285714284, "grad_norm": 0.21161924302577972, "kl": 0.0260009765625, "lambda_div_used": 0.6475020051002502, "learning_rate": 2.2629708984760706e-07, "loss": 0.0229, "reward": 0.20138549618422985, "reward_after_mean": 0.20138549618422985, "reward_after_std": 0.7262696456164122, "reward_before_mean": 0.6235175374895334, "reward_before_std": 0.6920611103996634, "reward_change_max": 0.0, "reward_change_mean": -0.42213201709091663, "reward_change_min": -0.6924243085086346, "reward_change_std": 0.2770402291789651, "reward_std": 0.7262696456164122, "rewards/accuracy_reward": 0.43750000931322575, "rewards/cosine_scaled_reward": 0.18601750303059816, "step": 391 }, { "clip_fraction": 0.0, "completion_length": 2778.979202270508, "epoch": 0.448, "grad_norm": 0.21174630522727966, "kl": 0.04703521728515625, "lambda_div_used": 0.6002580970525742, "learning_rate": 2.2412266235313973e-07, "loss": 0.0475, "reward": -0.2335773315280676, "reward_after_mean": -0.2335773315280676, "reward_after_std": 0.4885574597865343, "reward_before_mean": 0.04470384493470192, "reward_before_std": 0.4608147880062461, "reward_change_max": 0.0, "reward_change_mean": -0.2782811690121889, "reward_change_min": -0.5047546066343784, "reward_change_std": 0.18592887185513973, "reward_std": 0.4885574821382761, "rewards/accuracy_reward": 0.1875000074505806, "rewards/cosine_scaled_reward": -0.14279617369174957, "step": 392 }, { "clip_fraction": 0.0, "completion_length": 3245.291748046875, "epoch": 0.4491428571428571, "grad_norm": 0.23555870354175568, "kl": 0.02269744873046875, "lambda_div_used": 0.6642725691199303, "learning_rate": 2.2196411766036487e-07, "loss": 0.0778, "reward": 0.037709182128310204, "reward_after_mean": 0.037709182128310204, "reward_after_std": 0.7798551917076111, "reward_before_mean": 0.3366864509880543, "reward_before_std": 0.7692165970802307, "reward_change_max": 0.0, "reward_change_mean": -0.2989772502332926, "reward_change_min": -0.5559388622641563, "reward_change_std": 0.2134937485679984, "reward_std": 0.7798552140593529, "rewards/accuracy_reward": 0.2916666753590107, "rewards/cosine_scaled_reward": 0.04501976663595997, "step": 393 }, { "clip_fraction": 0.0, "completion_length": 3506.375030517578, "epoch": 0.4502857142857143, "grad_norm": 0.20107175409793854, "kl": 0.02321624755859375, "lambda_div_used": 0.5773903280496597, "learning_rate": 2.1982156097370557e-07, "loss": 0.0045, "reward": -0.3770834943279624, "reward_after_mean": -0.3770834943279624, "reward_after_std": 0.4104622136801481, "reward_before_mean": -0.1163465864956379, "reward_before_std": 0.3505119029432535, "reward_change_max": 0.0, "reward_change_mean": -0.26073691062629223, "reward_change_min": -0.4294714592397213, "reward_change_std": 0.15247186087071896, "reward_std": 0.4104622155427933, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.1788465972058475, "step": 394 }, { "clip_fraction": 0.0, "completion_length": 2758.2708435058594, "epoch": 0.4514285714285714, "grad_norm": 0.25962361693382263, "kl": 0.02793121337890625, "lambda_div_used": 0.540803536772728, "learning_rate": 2.1769509671835223e-07, "loss": -0.0008, "reward": -0.37039968371391296, "reward_after_mean": -0.37039968371391296, "reward_after_std": 0.2957852017134428, "reward_before_mean": -0.019847970455884933, "reward_before_std": 0.17921008728444576, "reward_change_max": 0.0, "reward_change_mean": -0.3505517225712538, "reward_change_min": -0.4961233139038086, "reward_change_std": 0.18551153503358364, "reward_std": 0.29578521102666855, "rewards/accuracy_reward": 0.125, "rewards/cosine_scaled_reward": -0.14484797231853008, "step": 395 }, { "clip_fraction": 0.0, "completion_length": 3528.8958740234375, "epoch": 0.45257142857142857, "grad_norm": 0.1785517930984497, "kl": 0.02210235595703125, "lambda_div_used": 0.620464064180851, "learning_rate": 2.1558482853517253e-07, "loss": 0.0166, "reward": -0.1592980690766126, "reward_after_mean": -0.1592980690766126, "reward_after_std": 0.5847718045115471, "reward_before_mean": 0.1181712131947279, "reward_before_std": 0.5596625525504351, "reward_change_max": 0.0, "reward_change_mean": -0.2774692941457033, "reward_change_min": -0.49834443628787994, "reward_change_std": 0.1908964067697525, "reward_std": 0.5847718250006437, "rewards/accuracy_reward": 0.20833333767950535, "rewards/cosine_scaled_reward": -0.0901621226221323, "step": 396 }, { "clip_fraction": 0.0, "completion_length": 3282.0416870117188, "epoch": 0.45371428571428574, "grad_norm": 0.17488232254981995, "kl": 0.0253448486328125, "lambda_div_used": 0.6301343515515327, "learning_rate": 2.134908592756607e-07, "loss": 0.0189, "reward": -0.1027121227234602, "reward_after_mean": -0.1027121227234602, "reward_after_std": 0.6213333550840616, "reward_before_mean": 0.1855018688365817, "reward_before_std": 0.6081824731081724, "reward_change_max": 0.0, "reward_change_mean": -0.2882139813154936, "reward_change_min": -0.5633565708994865, "reward_change_std": 0.20659327879548073, "reward_std": 0.6213333830237389, "rewards/accuracy_reward": 0.2083333395421505, "rewards/cosine_scaled_reward": -0.022831467911601067, "step": 397 }, { "clip_fraction": 0.0, "completion_length": 3207.0208740234375, "epoch": 0.45485714285714285, "grad_norm": 0.1854589581489563, "kl": 0.02447509765625, "lambda_div_used": 0.6168202310800552, "learning_rate": 2.1141329099692406e-07, "loss": 0.0535, "reward": -0.2422007992863655, "reward_after_mean": -0.2422007992863655, "reward_after_std": 0.5886733401566744, "reward_before_mean": 0.00447947415523231, "reward_before_std": 0.5368841402232647, "reward_change_max": 0.0, "reward_change_mean": -0.24668025597929955, "reward_change_min": -0.3933805041015148, "reward_change_std": 0.14942416176199913, "reward_std": 0.588673347607255, "rewards/accuracy_reward": 0.14583333767950535, "rewards/cosine_scaled_reward": -0.14135387679561973, "step": 398 }, { "clip_fraction": 0.0, "completion_length": 2977.979248046875, "epoch": 0.456, "grad_norm": 0.23114340007305145, "kl": 0.02495574951171875, "lambda_div_used": 0.6608658656477928, "learning_rate": 2.0935222495670968e-07, "loss": 0.0571, "reward": 0.1466597244143486, "reward_after_mean": 0.1466597244143486, "reward_after_std": 0.7965238057076931, "reward_before_mean": 0.523397309705615, "reward_before_std": 0.7609494812786579, "reward_change_max": 0.0, "reward_change_mean": -0.3767375871539116, "reward_change_min": -0.7153054252266884, "reward_change_std": 0.27144837006926537, "reward_std": 0.7965238224714994, "rewards/accuracy_reward": 0.3958333358168602, "rewards/cosine_scaled_reward": 0.1275639438536018, "step": 399 }, { "clip_fraction": 0.0, "completion_length": 2534.27091217041, "epoch": 0.45714285714285713, "grad_norm": 0.19662317633628845, "kl": 0.03588104248046875, "lambda_div_used": 0.6400952041149139, "learning_rate": 2.0730776160846853e-07, "loss": 0.0031, "reward": 0.17024126928299665, "reward_after_mean": 0.17024126928299665, "reward_after_std": 0.7864860761910677, "reward_before_mean": 0.6238485770300031, "reward_before_std": 0.649463377892971, "reward_change_max": 0.0, "reward_change_mean": -0.453607315197587, "reward_change_min": -0.6769693307578564, "reward_change_std": 0.2613063072785735, "reward_std": 0.786486092954874, "rewards/accuracy_reward": 0.43750000558793545, "rewards/cosine_scaled_reward": 0.18634857051074505, "step": 400 }, { "clip_fraction": 0.0, "completion_length": 3301.7083435058594, "epoch": 0.4582857142857143, "grad_norm": 0.17931640148162842, "kl": 0.01856231689453125, "lambda_div_used": 0.5612837076187134, "learning_rate": 2.0528000059645995e-07, "loss": 0.0208, "reward": -0.22075921669602394, "reward_after_mean": -0.22075921669602394, "reward_after_std": 0.383612310513854, "reward_before_mean": 0.16295179910957813, "reward_before_std": 0.2798503702506423, "reward_change_max": 0.0, "reward_change_mean": -0.38371098414063454, "reward_change_min": -0.5578874610364437, "reward_change_std": 0.21464537270367146, "reward_std": 0.38361233100295067, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.04538155533373356, "step": 401 }, { "clip_fraction": 0.0, "completion_length": 2953.854202270508, "epoch": 0.4594285714285714, "grad_norm": 0.22257892787456512, "kl": 0.03985595703125, "lambda_div_used": 0.5528981015086174, "learning_rate": 2.032690407508949e-07, "loss": -0.0342, "reward": -0.32120185531675816, "reward_after_mean": -0.32120185531675816, "reward_after_std": 0.35914880223572254, "reward_before_mean": 0.03579988703131676, "reward_before_std": 0.23527985624969006, "reward_change_max": 0.0, "reward_change_mean": -0.3570017348974943, "reward_change_min": -0.5012487955391407, "reward_change_std": 0.18728046771138906, "reward_std": 0.359148807823658, "rewards/accuracy_reward": 0.14583333395421505, "rewards/cosine_scaled_reward": -0.11003347486257553, "step": 402 }, { "clip_fraction": 0.0, "completion_length": 2653.5833587646484, "epoch": 0.4605714285714286, "grad_norm": 0.3024817705154419, "kl": 0.0894622802734375, "lambda_div_used": 0.5592624396085739, "learning_rate": 2.0127498008311922e-07, "loss": 0.0367, "reward": -0.20915501564741135, "reward_after_mean": -0.20915501564741135, "reward_after_std": 0.3597847409546375, "reward_before_mean": 0.18706247676163912, "reward_before_std": 0.26524644531309605, "reward_change_max": 0.0, "reward_change_mean": -0.3962174840271473, "reward_change_min": -0.5690255984663963, "reward_change_std": 0.21881575975567102, "reward_std": 0.3597847502678633, "rewards/accuracy_reward": 0.2291666716337204, "rewards/cosine_scaled_reward": -0.04210419673472643, "step": 403 }, { "clip_fraction": 0.0, "completion_length": 2970.000030517578, "epoch": 0.4617142857142857, "grad_norm": 0.2191041111946106, "kl": 0.07218170166015625, "lambda_div_used": 0.55616445094347, "learning_rate": 1.9929791578083655e-07, "loss": -0.0118, "reward": -0.20070256292819977, "reward_after_mean": -0.20070256292819977, "reward_after_std": 0.3485571127384901, "reward_before_mean": 0.21119800209999084, "reward_before_std": 0.24948583729565144, "reward_change_max": 0.0, "reward_change_mean": -0.4119005799293518, "reward_change_min": -0.5820152349770069, "reward_change_std": 0.22456652019172907, "reward_std": 0.348557123914361, "rewards/accuracy_reward": 0.2291666716337204, "rewards/cosine_scaled_reward": -0.01796867325901985, "step": 404 }, { "clip_fraction": 0.0, "completion_length": 2777.125030517578, "epoch": 0.46285714285714286, "grad_norm": 0.2112227976322174, "kl": 0.026214599609375, "lambda_div_used": 0.6189108490943909, "learning_rate": 1.9733794420337213e-07, "loss": -0.0015, "reward": 0.04253794066607952, "reward_after_mean": 0.04253794066607952, "reward_after_std": 0.6161006651818752, "reward_before_mean": 0.4405675707384944, "reward_before_std": 0.5447132457047701, "reward_change_max": 0.0, "reward_change_mean": -0.3980296514928341, "reward_change_min": -0.5984650664031506, "reward_change_std": 0.23571301344782114, "reward_std": 0.616100687533617, "rewards/accuracy_reward": 0.3541666753590107, "rewards/cosine_scaled_reward": 0.08640091214329004, "step": 405 }, { "clip_fraction": 0.0, "completion_length": 3016.2084045410156, "epoch": 0.464, "grad_norm": 0.20116589963436127, "kl": 0.0250701904296875, "lambda_div_used": 0.6647431552410126, "learning_rate": 1.9539516087697517e-07, "loss": 0.0383, "reward": -0.05482893157750368, "reward_after_mean": -0.05482893157750368, "reward_after_std": 0.7846493162214756, "reward_before_mean": 0.19566256801772397, "reward_before_std": 0.7684222515672445, "reward_change_max": 0.0, "reward_change_mean": -0.250491488724947, "reward_change_min": -0.4682968705892563, "reward_change_std": 0.18149811122566462, "reward_std": 0.784649346023798, "rewards/accuracy_reward": 0.25000000558793545, "rewards/cosine_scaled_reward": -0.05433744378387928, "step": 406 }, { "clip_fraction": 0.0, "completion_length": 2863.5625610351562, "epoch": 0.46514285714285714, "grad_norm": 0.18798598647117615, "kl": 0.02091217041015625, "lambda_div_used": 0.6001021265983582, "learning_rate": 1.934696604901642e-07, "loss": 0.009, "reward": -0.038052873685956, "reward_after_mean": -0.038052873685956, "reward_after_std": 0.5454999078065157, "reward_before_mean": 0.36550163105130196, "reward_before_std": 0.45983228739351034, "reward_change_max": 0.0, "reward_change_mean": -0.40355453081429005, "reward_change_min": -0.6564962938427925, "reward_change_std": 0.24441845808178186, "reward_std": 0.545499924570322, "rewards/accuracy_reward": 0.3125000074505806, "rewards/cosine_scaled_reward": 0.05300162732601166, "step": 407 }, { "clip_fraction": 0.0, "completion_length": 3096.729217529297, "epoch": 0.4662857142857143, "grad_norm": 0.1820216178894043, "kl": 0.02387237548828125, "lambda_div_used": 0.6040822714567184, "learning_rate": 1.915615368891117e-07, "loss": 0.0462, "reward": -0.030909111723303795, "reward_after_mean": -0.030909111723303795, "reward_after_std": 0.5603178422898054, "reward_before_mean": 0.36619370244443417, "reward_before_std": 0.4822367588058114, "reward_change_max": 0.0, "reward_change_mean": -0.3971028309315443, "reward_change_min": -0.6493563130497932, "reward_change_std": 0.24474582262337208, "reward_std": 0.5603178590536118, "rewards/accuracy_reward": 0.3125000074505806, "rewards/cosine_scaled_reward": 0.05369369266554713, "step": 408 }, { "clip_fraction": 0.0, "completion_length": 3487.2083435058594, "epoch": 0.4674285714285714, "grad_norm": 0.1749817430973053, "kl": 0.022735595703125, "lambda_div_used": 0.5657407864928246, "learning_rate": 1.8967088307307e-07, "loss": 0.0239, "reward": -0.3720417730510235, "reward_after_mean": -0.3720417730510235, "reward_after_std": 0.3428829610347748, "reward_before_mean": -0.09223516099154949, "reward_before_std": 0.2999506648629904, "reward_change_max": 0.0, "reward_change_mean": -0.2798066083341837, "reward_change_min": -0.44451572000980377, "reward_change_std": 0.1677148537710309, "reward_std": 0.34288296662271023, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.1755684930831194, "step": 409 }, { "clip_fraction": 0.0, "completion_length": 3164.0208435058594, "epoch": 0.4685714285714286, "grad_norm": 0.20155300199985504, "kl": 0.0316925048828125, "lambda_div_used": 0.6230098828673363, "learning_rate": 1.8779779118983867e-07, "loss": 0.0083, "reward": -0.21067720837891102, "reward_after_mean": -0.21067720837891102, "reward_after_std": 0.6115751005709171, "reward_before_mean": 0.051790774799883366, "reward_before_std": 0.5689785033464432, "reward_change_max": 0.0, "reward_change_mean": -0.26246798411011696, "reward_change_min": -0.4530046824365854, "reward_change_std": 0.17007357813417912, "reward_std": 0.6115751080214977, "rewards/accuracy_reward": 0.1458333358168602, "rewards/cosine_scaled_reward": -0.09404256660491228, "step": 410 }, { "clip_fraction": 0.0, "completion_length": 3481.5208435058594, "epoch": 0.4697142857142857, "grad_norm": 0.17682720720767975, "kl": 0.0176239013671875, "lambda_div_used": 0.5787669941782951, "learning_rate": 1.8594235253127372e-07, "loss": 0.0131, "reward": -0.24107754603028297, "reward_after_mean": -0.24107754603028297, "reward_after_std": 0.403771610930562, "reward_before_mean": 0.08782530669122934, "reward_before_std": 0.3601356018334627, "reward_change_max": 0.0, "reward_change_mean": -0.3289028462022543, "reward_change_min": -0.538174994289875, "reward_change_std": 0.1987251853570342, "reward_std": 0.4037716370075941, "rewards/accuracy_reward": 0.1458333395421505, "rewards/cosine_scaled_reward": -0.05800804868340492, "step": 411 }, { "clip_fraction": 0.0, "completion_length": 3293.9583587646484, "epoch": 0.47085714285714286, "grad_norm": 0.20358195900917053, "kl": 0.0255584716796875, "lambda_div_used": 0.6070855334401131, "learning_rate": 1.8410465752883758e-07, "loss": 0.0236, "reward": -0.2317246664315462, "reward_after_mean": -0.2317246664315462, "reward_after_std": 0.5295515451580286, "reward_before_mean": 0.04360375925898552, "reward_before_std": 0.4952409639954567, "reward_change_max": 0.0, "reward_change_mean": -0.27532841823995113, "reward_change_min": -0.4982520490884781, "reward_change_std": 0.17953196447342634, "reward_std": 0.5295515581965446, "rewards/accuracy_reward": 0.12500000186264515, "rewards/cosine_scaled_reward": -0.08139624074101448, "step": 412 }, { "clip_fraction": 0.0, "completion_length": 3196.8958740234375, "epoch": 0.472, "grad_norm": 0.17003904283046722, "kl": 0.02420806884765625, "lambda_div_used": 0.5897047892212868, "learning_rate": 1.822847957491922e-07, "loss": 0.0277, "reward": -0.15652739070355892, "reward_after_mean": -0.15652739070355892, "reward_after_std": 0.4894407894462347, "reward_before_mean": 0.21204309538006783, "reward_before_std": 0.41512753814458847, "reward_change_max": 0.0, "reward_change_mean": -0.3685704581439495, "reward_change_min": -0.6019548028707504, "reward_change_std": 0.22615858726203442, "reward_std": 0.48944081366062164, "rewards/accuracy_reward": 0.2291666679084301, "rewards/cosine_scaled_reward": -0.017123591154813766, "step": 413 }, { "clip_fraction": 0.0, "completion_length": 3567.9791870117188, "epoch": 0.47314285714285714, "grad_norm": 0.1660469025373459, "kl": 0.02222442626953125, "lambda_div_used": 0.5847119837999344, "learning_rate": 1.804828558898332e-07, "loss": -0.0021, "reward": -0.37159527838230133, "reward_after_mean": -0.37159527838230133, "reward_after_std": 0.43851041980087757, "reward_before_mean": -0.11719285417348146, "reward_before_std": 0.38712745532393456, "reward_change_max": 0.0, "reward_change_mean": -0.2544024232774973, "reward_change_min": -0.4335598982870579, "reward_change_std": 0.15531378239393234, "reward_std": 0.4385104477405548, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.17969285137951374, "step": 414 }, { "clip_fraction": 0.0, "completion_length": 3518.5625, "epoch": 0.4742857142857143, "grad_norm": 0.17121130228042603, "kl": 0.01818084716796875, "lambda_div_used": 0.5783144906163216, "learning_rate": 1.7869892577476722e-07, "loss": 0.0157, "reward": -0.3618303295224905, "reward_after_mean": -0.3618303295224905, "reward_after_std": 0.41689756140112877, "reward_before_mean": -0.09457835368812084, "reward_before_std": 0.35964186675846577, "reward_change_max": 0.0, "reward_change_mean": -0.267251955345273, "reward_change_min": -0.430114571005106, "reward_change_std": 0.15875060949474573, "reward_std": 0.41689757257699966, "rewards/accuracy_reward": 0.08333333395421505, "rewards/cosine_scaled_reward": -0.17791170440614223, "step": 415 }, { "clip_fraction": 0.0, "completion_length": 2879.7708892822266, "epoch": 0.4754285714285714, "grad_norm": 0.19891639053821564, "kl": 0.03377532958984375, "lambda_div_used": 0.6242444291710854, "learning_rate": 1.7693309235023127e-07, "loss": 0.0013, "reward": -0.09552648849785328, "reward_after_mean": -0.09552648849785328, "reward_after_std": 0.604328878223896, "reward_before_mean": 0.20598628465086222, "reward_before_std": 0.581903294660151, "reward_change_max": 0.0, "reward_change_mean": -0.30151278898119926, "reward_change_min": -0.5268363878130913, "reward_change_std": 0.20368388202041388, "reward_std": 0.6043288819491863, "rewards/accuracy_reward": 0.22916667349636555, "rewards/cosine_scaled_reward": -0.023180373944342136, "step": 416 }, { "clip_fraction": 0.0, "completion_length": 3474.416717529297, "epoch": 0.4765714285714286, "grad_norm": 0.20228314399719238, "kl": 0.0251007080078125, "lambda_div_used": 0.5665470510721207, "learning_rate": 1.7518544168045524e-07, "loss": 0.0052, "reward": -0.40818358585238457, "reward_after_mean": -0.40818358585238457, "reward_after_std": 0.34758291579782963, "reward_before_mean": -0.1455591917037964, "reward_before_std": 0.30561812967061996, "reward_change_max": 0.0, "reward_change_mean": -0.2626244034618139, "reward_change_min": -0.43587180972099304, "reward_change_std": 0.16136188618838787, "reward_std": 0.3475829176604748, "rewards/accuracy_reward": 0.0625, "rewards/cosine_scaled_reward": -0.20805918611586094, "step": 417 }, { "clip_fraction": 0.0, "completion_length": 2763.3333892822266, "epoch": 0.4777142857142857, "grad_norm": 0.2209230661392212, "kl": 0.05419158935546875, "lambda_div_used": 0.6229055225849152, "learning_rate": 1.7345605894346726e-07, "loss": 0.0448, "reward": 0.019997593015432358, "reward_after_mean": 0.019997593015432358, "reward_after_std": 0.6416514366865158, "reward_before_mean": 0.4030896439217031, "reward_before_std": 0.5654843933880329, "reward_change_max": 0.0, "reward_change_mean": -0.3830920699983835, "reward_change_min": -0.5921037830412388, "reward_change_std": 0.22725382912904024, "reward_std": 0.6416514590382576, "rewards/accuracy_reward": 0.3333333395421505, "rewards/cosine_scaled_reward": 0.06975632207468152, "step": 418 }, { "clip_fraction": 0.0, "completion_length": 3385.000030517578, "epoch": 0.47885714285714287, "grad_norm": 0.20651023089885712, "kl": 0.03195953369140625, "lambda_div_used": 0.5702432468533516, "learning_rate": 1.7174502842694212e-07, "loss": 0.0502, "reward": -0.12258239835500717, "reward_after_mean": -0.12258239835500717, "reward_after_std": 0.42125734128057957, "reward_before_mean": 0.29360441863536835, "reward_before_std": 0.3158922381699085, "reward_change_max": 0.0, "reward_change_mean": -0.4161868281662464, "reward_change_min": -0.5788598395884037, "reward_change_std": 0.22479810379445553, "reward_std": 0.4212573431432247, "rewards/accuracy_reward": 0.25000000558793545, "rewards/cosine_scaled_reward": 0.04360440792515874, "step": 419 }, { "clip_fraction": 0.0, "completion_length": 2854.354202270508, "epoch": 0.48, "grad_norm": 0.2658880352973938, "kl": 0.03424072265625, "lambda_div_used": 0.5758904963731766, "learning_rate": 1.7005243352409333e-07, "loss": 0.0295, "reward": -0.1418533055111766, "reward_after_mean": -0.1418533055111766, "reward_after_std": 0.44697228260338306, "reward_before_mean": 0.25108543410897255, "reward_before_std": 0.3429322447627783, "reward_change_max": 0.0, "reward_change_mean": -0.3929387368261814, "reward_change_min": -0.5611292459070683, "reward_change_std": 0.21301832795143127, "reward_std": 0.44697230495512486, "rewards/accuracy_reward": 0.25000000558793545, "rewards/cosine_scaled_reward": 0.0010854215361177921, "step": 420 }, { "clip_fraction": 0.0, "completion_length": 3553.9166870117188, "epoch": 0.48114285714285715, "grad_norm": 0.19543302059173584, "kl": 0.02838897705078125, "lambda_div_used": 0.5882885977625847, "learning_rate": 1.6837835672960831e-07, "loss": 0.0069, "reward": -0.31190643832087517, "reward_after_mean": -0.31190643832087517, "reward_after_std": 0.433170011267066, "reward_before_mean": -0.03402327001094818, "reward_before_std": 0.40585840679705143, "reward_change_max": 0.0, "reward_change_mean": -0.27788317389786243, "reward_change_min": -0.4925220459699631, "reward_change_std": 0.18238117825239897, "reward_std": 0.43317001312971115, "rewards/accuracy_reward": 0.1041666679084301, "rewards/cosine_scaled_reward": -0.13818994909524918, "step": 421 }, { "clip_fraction": 0.0, "completion_length": 3206.8333740234375, "epoch": 0.48228571428571426, "grad_norm": 0.19716881215572357, "kl": 0.02154541015625, "lambda_div_used": 0.5794828534126282, "learning_rate": 1.6672287963562852e-07, "loss": 0.0041, "reward": -0.22385949455201626, "reward_after_mean": -0.22385949455201626, "reward_after_std": 0.46671467646956444, "reward_before_mean": 0.1158480579033494, "reward_before_std": 0.3668345585465431, "reward_change_max": 0.0, "reward_change_mean": -0.3397075552493334, "reward_change_min": -0.5031177438795567, "reward_change_std": 0.19169152900576591, "reward_std": 0.46671469137072563, "rewards/accuracy_reward": 0.20833333395421505, "rewards/cosine_scaled_reward": -0.09248528070747852, "step": 422 }, { "clip_fraction": 0.0, "completion_length": 3133.250030517578, "epoch": 0.48342857142857143, "grad_norm": 0.22129932045936584, "kl": 0.05303955078125, "lambda_div_used": 0.6031422987580299, "learning_rate": 1.6508608292777203e-07, "loss": 0.0528, "reward": -0.15305102244019508, "reward_after_mean": -0.15305102244019508, "reward_after_std": 0.5137906111776829, "reward_before_mean": 0.15428910590708256, "reward_before_std": 0.4784602206200361, "reward_change_max": 0.0, "reward_change_mean": -0.30734013952314854, "reward_change_min": -0.48346148803830147, "reward_change_std": 0.19110194873064756, "reward_std": 0.5137906298041344, "rewards/accuracy_reward": 0.18750000558793545, "rewards/cosine_scaled_reward": -0.03321088245138526, "step": 423 }, { "clip_fraction": 0.0, "completion_length": 3403.0208435058594, "epoch": 0.4845714285714286, "grad_norm": 0.2508692145347595, "kl": 0.02907562255859375, "lambda_div_used": 0.6078787297010422, "learning_rate": 1.6346804638120098e-07, "loss": 0.0337, "reward": -0.27756378054618835, "reward_after_mean": -0.27756378054618835, "reward_after_std": 0.5265482496470213, "reward_before_mean": -0.022029063664376736, "reward_before_std": 0.5001667328178883, "reward_change_max": 0.0, "reward_change_mean": -0.25553470477461815, "reward_change_min": -0.46236270293593407, "reward_change_std": 0.1711604781448841, "reward_std": 0.5265482757240534, "rewards/accuracy_reward": 0.12500000186264515, "rewards/cosine_scaled_reward": -0.14702906738966703, "step": 424 }, { "clip_fraction": 0.0, "completion_length": 2564.3334197998047, "epoch": 0.4857142857142857, "grad_norm": 0.20480872690677643, "kl": 0.0807342529296875, "lambda_div_used": 0.6461614891886711, "learning_rate": 1.6186884885673413e-07, "loss": 0.0375, "reward": 0.26849367283284664, "reward_after_mean": 0.26849367283284664, "reward_after_std": 0.7015949115157127, "reward_before_mean": 0.7275500744581223, "reward_before_std": 0.6788945775479078, "reward_change_max": 0.0, "reward_change_mean": -0.4590563476085663, "reward_change_min": -0.7770565748214722, "reward_change_std": 0.30189999006688595, "reward_std": 0.7015949375927448, "rewards/accuracy_reward": 0.5000000149011612, "rewards/cosine_scaled_reward": 0.2275500291143544, "step": 425 }, { "clip_fraction": 0.0, "completion_length": 2978.2708740234375, "epoch": 0.4868571428571429, "grad_norm": 0.2312241494655609, "kl": 0.03387451171875, "lambda_div_used": 0.6494922637939453, "learning_rate": 1.6028856829700258e-07, "loss": 0.0463, "reward": 0.06004667468369007, "reward_after_mean": 0.06004667468369007, "reward_after_std": 0.6850448977202177, "reward_before_mean": 0.3802625238895416, "reward_before_std": 0.7007070239633322, "reward_change_max": 0.0, "reward_change_mean": -0.3202158473432064, "reward_change_min": -0.5520904809236526, "reward_change_std": 0.228487653657794, "reward_std": 0.6850449126213789, "rewards/accuracy_reward": 0.33333334513008595, "rewards/cosine_scaled_reward": 0.046929189818911254, "step": 426 }, { "clip_fraction": 0.0, "completion_length": 3398.875030517578, "epoch": 0.488, "grad_norm": 0.21663960814476013, "kl": 0.0270843505859375, "lambda_div_used": 0.5862439498305321, "learning_rate": 1.5872728172265146e-07, "loss": 0.0423, "reward": -0.1549149975180626, "reward_after_mean": -0.1549149975180626, "reward_after_std": 0.49510751478374004, "reward_before_mean": 0.22673227824270725, "reward_before_std": 0.4013592656701803, "reward_change_max": 0.0, "reward_change_mean": -0.3816472813487053, "reward_change_min": -0.6057535372674465, "reward_change_std": 0.2297013308852911, "reward_std": 0.49510751850903034, "rewards/accuracy_reward": 0.2291666679084301, "rewards/cosine_scaled_reward": -0.0024343752302229404, "step": 427 }, { "clip_fraction": 0.0, "completion_length": 3370.8125610351562, "epoch": 0.48914285714285716, "grad_norm": 0.19991178810596466, "kl": 0.02994537353515625, "lambda_div_used": 0.5620873495936394, "learning_rate": 1.5718506522858572e-07, "loss": 0.0374, "reward": -0.4467948842793703, "reward_after_mean": -0.4467948842793703, "reward_after_std": 0.3462687749415636, "reward_before_mean": -0.19363035261631012, "reward_before_std": 0.2834156332537532, "reward_change_max": 0.0, "reward_change_mean": -0.25316453725099564, "reward_change_min": -0.4254877343773842, "reward_change_std": 0.15061054658144712, "reward_std": 0.3462687823921442, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.23529702005907893, "step": 428 }, { "clip_fraction": 0.0, "completion_length": 2884.916717529297, "epoch": 0.49028571428571427, "grad_norm": 0.2918151319026947, "kl": 0.032806396484375, "lambda_div_used": 0.6370875462889671, "learning_rate": 1.5566199398026147e-07, "loss": 0.0928, "reward": -0.0687477570027113, "reward_after_mean": -0.0687477570027113, "reward_after_std": 0.6580539904534817, "reward_before_mean": 0.22279318794608116, "reward_before_std": 0.6305141560733318, "reward_change_max": 0.0, "reward_change_mean": -0.29154095612466335, "reward_change_min": -0.5219893492758274, "reward_change_std": 0.19429981894791126, "reward_std": 0.6580540053546429, "rewards/accuracy_reward": 0.2500000074505806, "rewards/cosine_scaled_reward": -0.02720680704806, "step": 429 }, { "clip_fraction": 0.0, "completion_length": 2888.062545776367, "epoch": 0.49142857142857144, "grad_norm": 0.24667245149612427, "kl": 0.0274505615234375, "lambda_div_used": 0.5822442546486855, "learning_rate": 1.5415814221002265e-07, "loss": 0.0627, "reward": -0.1749287759885192, "reward_after_mean": -0.1749287759885192, "reward_after_std": 0.4813099801540375, "reward_before_mean": 0.19111618725582957, "reward_before_std": 0.3792247697710991, "reward_change_max": 0.0, "reward_change_mean": -0.36604496091604233, "reward_change_min": -0.5452068485319614, "reward_change_std": 0.20586189348250628, "reward_std": 0.48130999878048897, "rewards/accuracy_reward": 0.22916666977107525, "rewards/cosine_scaled_reward": -0.03805047646164894, "step": 430 }, { "clip_fraction": 0.0, "completion_length": 2943.312545776367, "epoch": 0.49257142857142855, "grad_norm": 0.1868581473827362, "kl": 0.0248260498046875, "lambda_div_used": 0.5592194423079491, "learning_rate": 1.5267358321348285e-07, "loss": 0.0178, "reward": -0.33158062398433685, "reward_after_mean": -0.33158062398433685, "reward_after_std": 0.38170251809060574, "reward_before_mean": 0.005494551733136177, "reward_before_std": 0.2644110005348921, "reward_change_max": 0.0, "reward_change_mean": -0.3370751831680536, "reward_change_min": -0.49553232640028, "reward_change_std": 0.17910714354366064, "reward_std": 0.38170253671705723, "rewards/accuracy_reward": 0.14583333395421505, "rewards/cosine_scaled_reward": -0.14033876918256283, "step": 431 }, { "clip_fraction": 0.0, "completion_length": 3327.312545776367, "epoch": 0.4937142857142857, "grad_norm": 0.1837572157382965, "kl": 0.030048370361328125, "lambda_div_used": 0.6207441538572311, "learning_rate": 1.5120838934595337e-07, "loss": 0.0018, "reward": -0.11540279164910316, "reward_after_mean": -0.11540279164910316, "reward_after_std": 0.6543992068618536, "reward_before_mean": 0.2164886794053018, "reward_before_std": 0.5563324019312859, "reward_change_max": 0.0, "reward_change_mean": -0.3318914845585823, "reward_change_min": -0.5126226246356964, "reward_change_std": 0.1903834594413638, "reward_std": 0.6543992217630148, "rewards/accuracy_reward": 0.22916666977107525, "rewards/cosine_scaled_reward": -0.012677978374995291, "step": 432 }, { "clip_fraction": 0.0, "completion_length": 3244.6666870117188, "epoch": 0.4948571428571429, "grad_norm": 0.17697474360466003, "kl": 0.02024078369140625, "lambda_div_used": 0.5808933079242706, "learning_rate": 1.4976263201891613e-07, "loss": 0.0253, "reward": -0.21334804315119982, "reward_after_mean": -0.21334804315119982, "reward_after_std": 0.4726564548909664, "reward_before_mean": 0.14756887964904308, "reward_before_std": 0.3693957943469286, "reward_change_max": 0.0, "reward_change_mean": -0.3609169237315655, "reward_change_min": -0.5521964095532894, "reward_change_std": 0.20300101209431887, "reward_std": 0.472656462341547, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.03993113758042455, "step": 433 }, { "clip_fraction": 0.0, "completion_length": 3421.625030517578, "epoch": 0.496, "grad_norm": 0.20184823870658875, "kl": 0.02996826171875, "lambda_div_used": 0.5665369555354118, "learning_rate": 1.483363816965435e-07, "loss": 0.0347, "reward": -0.383462518453598, "reward_after_mean": -0.383462518453598, "reward_after_std": 0.3493436388671398, "reward_before_mean": -0.10589290224015713, "reward_before_std": 0.30595881305634975, "reward_change_max": 0.0, "reward_change_mean": -0.27756960690021515, "reward_change_min": -0.4418371096253395, "reward_change_std": 0.16845602542161942, "reward_std": 0.3493436388671398, "rewards/accuracy_reward": 0.0625, "rewards/cosine_scaled_reward": -0.16839290224015713, "step": 434 }, { "clip_fraction": 0.0, "completion_length": 2757.0416870117188, "epoch": 0.49714285714285716, "grad_norm": 0.2228793352842331, "kl": 0.036224365234375, "lambda_div_used": 0.5751429796218872, "learning_rate": 1.469297078922642e-07, "loss": -0.0206, "reward": -0.21108826622366905, "reward_after_mean": -0.21108826622366905, "reward_after_std": 0.3888908326625824, "reward_before_mean": 0.12317578494548798, "reward_before_std": 0.34171063639223576, "reward_change_max": 0.0, "reward_change_mean": -0.3342640623450279, "reward_change_min": -0.5097222179174423, "reward_change_std": 0.19784908555448055, "reward_std": 0.38889083825051785, "rewards/accuracy_reward": 0.1875000074505806, "rewards/cosine_scaled_reward": -0.06432422064244747, "step": 435 }, { "clip_fraction": 0.0, "completion_length": 2492.562515258789, "epoch": 0.4982857142857143, "grad_norm": 0.2213590145111084, "kl": 0.04131317138671875, "lambda_div_used": 0.5603080689907074, "learning_rate": 1.4554267916537495e-07, "loss": 0.0082, "reward": 0.10276253148913383, "reward_after_mean": 0.10276253148913383, "reward_after_std": 0.4578944016247988, "reward_before_mean": 0.6972460262477398, "reward_before_std": 0.2714722091332078, "reward_change_max": 0.0, "reward_change_mean": -0.5944835022091866, "reward_change_min": -0.8258654065430164, "reward_change_std": 0.31631586141884327, "reward_std": 0.45789442397654057, "rewards/accuracy_reward": 0.4583333358168602, "rewards/cosine_scaled_reward": 0.2389126904308796, "step": 436 }, { "clip_fraction": 0.0, "completion_length": 3545.125030517578, "epoch": 0.49942857142857144, "grad_norm": 0.2180529683828354, "kl": 0.03448486328125, "lambda_div_used": 0.5371564850211143, "learning_rate": 1.4417536311769885e-07, "loss": 0.0022, "reward": -0.5042365528643131, "reward_after_mean": -0.5042365528643131, "reward_after_std": 0.24860260263085365, "reward_before_mean": -0.22916061151772738, "reward_before_std": 0.16317266272380948, "reward_change_max": 0.0, "reward_change_mean": -0.27507593110203743, "reward_change_min": -0.3912927135825157, "reward_change_std": 0.14651282224804163, "reward_std": 0.24860260635614395, "rewards/accuracy_reward": 0.0, "rewards/cosine_scaled_reward": -0.22916060779243708, "step": 437 }, { "clip_fraction": 0.0, "completion_length": 3400.625030517578, "epoch": 0.5005714285714286, "grad_norm": 0.202320396900177, "kl": 0.01992034912109375, "lambda_div_used": 0.6293608918786049, "learning_rate": 1.4282782639029128e-07, "loss": -0.0007, "reward": -0.1771051762625575, "reward_after_mean": -0.1771051762625575, "reward_after_std": 0.6358457654714584, "reward_before_mean": 0.07997776940464973, "reward_before_std": 0.6038781059905887, "reward_change_max": 0.0, "reward_change_mean": -0.25708296336233616, "reward_change_min": -0.4915091060101986, "reward_change_std": 0.1804003519937396, "reward_std": 0.635845772922039, "rewards/accuracy_reward": 0.16666666977107525, "rewards/cosine_scaled_reward": -0.08668889198452234, "step": 438 }, { "clip_fraction": 0.0, "completion_length": 3058.7708740234375, "epoch": 0.5017142857142857, "grad_norm": 0.2478969246149063, "kl": 0.022705078125, "lambda_div_used": 0.5349839776754379, "learning_rate": 1.4150013466019114e-07, "loss": -0.006, "reward": -0.3505335934460163, "reward_after_mean": -0.3505335934460163, "reward_after_std": 0.28793079778552055, "reward_before_mean": 0.025686198845505714, "reward_before_std": 0.15359550900757313, "reward_change_max": 0.0, "reward_change_mean": -0.376219779253006, "reward_change_min": -0.5306564755737782, "reward_change_std": 0.1949494332075119, "reward_std": 0.28793080151081085, "rewards/accuracy_reward": 0.125, "rewards/cosine_scaled_reward": -0.09931380115449429, "step": 439 }, { "clip_fraction": 0.0, "completion_length": 3221.375045776367, "epoch": 0.5028571428571429, "grad_norm": 0.2263765186071396, "kl": 0.0279388427734375, "lambda_div_used": 0.5522914975881577, "learning_rate": 1.4019235263722034e-07, "loss": -0.0071, "reward": -0.32993070036172867, "reward_after_mean": -0.32993070036172867, "reward_after_std": 0.3648950830101967, "reward_before_mean": 0.019758086651563644, "reward_before_std": 0.23307441547513008, "reward_change_max": 0.0, "reward_change_mean": -0.3496888056397438, "reward_change_min": -0.4956108070909977, "reward_change_std": 0.18170258775353432, "reward_std": 0.3648951053619385, "rewards/accuracy_reward": 0.14583333395421505, "rewards/cosine_scaled_reward": -0.1260752510279417, "step": 440 }, { "clip_fraction": 0.0, "completion_length": 3295.1666870117188, "epoch": 0.504, "grad_norm": 0.20000971853733063, "kl": 0.03437042236328125, "lambda_div_used": 0.5897822231054306, "learning_rate": 1.3890454406082956e-07, "loss": 0.013, "reward": -0.19828267022967339, "reward_after_mean": -0.19828267022967339, "reward_after_std": 0.5012535229325294, "reward_before_mean": 0.14997220900841057, "reward_before_std": 0.41361709870398045, "reward_change_max": 0.0, "reward_change_mean": -0.34825489297509193, "reward_change_min": -0.5379728861153126, "reward_change_std": 0.2032255632802844, "reward_std": 0.50125353038311, "rewards/accuracy_reward": 0.20833333395421505, "rewards/cosine_scaled_reward": -0.058361124247312546, "step": 441 }, { "clip_fraction": 0.0, "completion_length": 3369.291717529297, "epoch": 0.5051428571428571, "grad_norm": 0.20546789467334747, "kl": 0.0328521728515625, "lambda_div_used": 0.5888748988509178, "learning_rate": 1.3763677169699217e-07, "loss": 0.012, "reward": -0.27872464805841446, "reward_after_mean": -0.27872464805841446, "reward_after_std": 0.4313396345824003, "reward_before_mean": 0.010555454529821873, "reward_before_std": 0.40844325441867113, "reward_change_max": 0.0, "reward_change_mean": -0.2892800811678171, "reward_change_min": -0.5037051253020763, "reward_change_std": 0.18890101090073586, "reward_std": 0.43133964389562607, "rewards/accuracy_reward": 0.1250000037252903, "rewards/cosine_scaled_reward": -0.11444455850869417, "step": 442 }, { "clip_fraction": 0.0, "completion_length": 3373.5834045410156, "epoch": 0.5062857142857143, "grad_norm": 0.18615411221981049, "kl": 0.0263214111328125, "lambda_div_used": 0.5995504036545753, "learning_rate": 1.3638909733514452e-07, "loss": 0.0087, "reward": -0.21670597605407238, "reward_after_mean": -0.21670597605407238, "reward_after_std": 0.49942680075764656, "reward_before_mean": 0.07735187746584415, "reward_before_std": 0.4567106030881405, "reward_change_max": 0.0, "reward_change_mean": -0.2940578758716583, "reward_change_min": -0.47536875680088997, "reward_change_std": 0.1786812385544181, "reward_std": 0.4994268212467432, "rewards/accuracy_reward": 0.16666667349636555, "rewards/cosine_scaled_reward": -0.0893147811293602, "step": 443 }, { "clip_fraction": 0.0, "completion_length": 3232.8958587646484, "epoch": 0.5074285714285715, "grad_norm": 0.18814043700695038, "kl": 0.02674102783203125, "lambda_div_used": 0.5554048493504524, "learning_rate": 1.351615817851748e-07, "loss": 0.0064, "reward": -0.364313580095768, "reward_after_mean": -0.364313580095768, "reward_after_std": 0.30347153544425964, "reward_before_mean": -0.06013181805610657, "reward_before_std": 0.24870464857667685, "reward_change_max": 0.0, "reward_change_mean": -0.3041817583143711, "reward_change_min": -0.452035766094923, "reward_change_std": 0.17346176877617836, "reward_std": 0.30347154662013054, "rewards/accuracy_reward": 0.1041666716337204, "rewards/cosine_scaled_reward": -0.16429849341511726, "step": 444 }, { "clip_fraction": 0.0, "completion_length": 3117.916717529297, "epoch": 0.5085714285714286, "grad_norm": 0.21639417111873627, "kl": 0.0280609130859375, "lambda_div_used": 0.6262663900852203, "learning_rate": 1.3395428487445914e-07, "loss": 0.0215, "reward": 0.00363805890083313, "reward_after_mean": 0.00363805890083313, "reward_after_std": 0.6511832810938358, "reward_before_mean": 0.3764121076092124, "reward_before_std": 0.5842630490660667, "reward_change_max": 0.0, "reward_change_mean": -0.37277403846383095, "reward_change_min": -0.6343755125999451, "reward_change_std": 0.23563686851412058, "reward_std": 0.6511833071708679, "rewards/accuracy_reward": 0.31250000558793545, "rewards/cosine_scaled_reward": 0.06391207501292229, "step": 445 }, { "clip_fraction": 0.0, "completion_length": 3189.5833587646484, "epoch": 0.5097142857142857, "grad_norm": 0.18836621940135956, "kl": 0.13494873046875, "lambda_div_used": 0.5945059955120087, "learning_rate": 1.3276726544494571e-07, "loss": -0.0141, "reward": -0.31021482590585947, "reward_after_mean": -0.31021482590585947, "reward_after_std": 0.5102730803191662, "reward_before_mean": -0.05426267348229885, "reward_before_std": 0.43475321121513844, "reward_change_max": 0.0, "reward_change_mean": -0.25595215894281864, "reward_change_min": -0.4035803750157356, "reward_change_std": 0.14673306699842215, "reward_std": 0.5102730933576822, "rewards/accuracy_reward": 0.0833333358168602, "rewards/cosine_scaled_reward": -0.1375960111618042, "step": 446 }, { "clip_fraction": 0.0, "completion_length": 2952.333366394043, "epoch": 0.5108571428571429, "grad_norm": 0.20803417265415192, "kl": 0.0406341552734375, "lambda_div_used": 0.5918032377958298, "learning_rate": 1.316005813502869e-07, "loss": -0.0004, "reward": -0.22545085521414876, "reward_after_mean": -0.22545085521414876, "reward_after_std": 0.4746465887874365, "reward_before_mean": 0.08152532763779163, "reward_before_std": 0.41641608998179436, "reward_change_max": 0.0, "reward_change_mean": -0.3069761786609888, "reward_change_min": -0.4963514022529125, "reward_change_std": 0.18319544661790133, "reward_std": 0.47464660182595253, "rewards/accuracy_reward": 0.16666667349636555, "rewards/cosine_scaled_reward": -0.0851413361961022, "step": 447 }, { "clip_fraction": 0.0, "completion_length": 2917.0833740234375, "epoch": 0.512, "grad_norm": 0.23971594870090485, "kl": 0.0425567626953125, "lambda_div_used": 0.572822131216526, "learning_rate": 1.3045428945301953e-07, "loss": 0.0604, "reward": -0.1545051522552967, "reward_after_mean": -0.1545051522552967, "reward_after_std": 0.43291558139026165, "reward_before_mean": 0.24294945178553462, "reward_before_std": 0.32697561848908663, "reward_change_max": 0.0, "reward_change_mean": -0.3974546194076538, "reward_change_min": -0.5821000263094902, "reward_change_std": 0.21722743939608335, "reward_std": 0.43291558511555195, "rewards/accuracy_reward": 0.25000000558793545, "rewards/cosine_scaled_reward": -0.007050560787320137, "step": 448 }, { "clip_fraction": 0.0, "completion_length": 2867.229202270508, "epoch": 0.5131428571428571, "grad_norm": 0.18195681273937225, "kl": 0.04132080078125, "lambda_div_used": 0.5599837079644203, "learning_rate": 1.2932844562179352e-07, "loss": 0.0192, "reward": -0.11188772693276405, "reward_after_mean": -0.11188772693276405, "reward_after_std": 0.428832221776247, "reward_before_mean": 0.36257917433977127, "reward_before_std": 0.2708895690739155, "reward_change_max": 0.0, "reward_change_mean": -0.4744668770581484, "reward_change_min": -0.6687535494565964, "reward_change_std": 0.25217378325760365, "reward_std": 0.4288322236388922, "rewards/accuracy_reward": 0.2916666679084301, "rewards/cosine_scaled_reward": 0.0709124926943332, "step": 449 }, { "clip_fraction": 0.0, "completion_length": 3147.9583740234375, "epoch": 0.5142857142857142, "grad_norm": 0.20482422411441803, "kl": 0.0301361083984375, "lambda_div_used": 0.5780885741114616, "learning_rate": 1.2822310472864885e-07, "loss": 0.0021, "reward": -0.2325735818594694, "reward_after_mean": -0.2325735818594694, "reward_after_std": 0.46496822871267796, "reward_before_mean": 0.12920803762972355, "reward_before_std": 0.3562629660591483, "reward_change_max": 0.0, "reward_change_mean": -0.3617816399782896, "reward_change_min": -0.5544614754617214, "reward_change_std": 0.204494365490973, "reward_std": 0.4649682566523552, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.0582919679582119, "step": 450 }, { "clip_fraction": 0.0, "completion_length": 3060.2083740234375, "epoch": 0.5154285714285715, "grad_norm": 0.24665038287639618, "kl": 0.03118133544921875, "lambda_div_used": 0.5588981136679649, "learning_rate": 1.2713832064634125e-07, "loss": 0.0018, "reward": -0.22303282469511032, "reward_after_mean": -0.22303282469511032, "reward_after_std": 0.37040900625288486, "reward_before_mean": 0.15426092967391014, "reward_before_std": 0.26707543432712555, "reward_change_max": 0.0, "reward_change_mean": -0.3772937189787626, "reward_change_min": -0.5537764392793179, "reward_change_std": 0.2077095191925764, "reward_std": 0.37040901742875576, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.0540724266320467, "step": 451 }, { "clip_fraction": 0.0, "completion_length": 3346.7708740234375, "epoch": 0.5165714285714286, "grad_norm": 0.17696413397789001, "kl": 0.02454376220703125, "lambda_div_used": 0.6324428245425224, "learning_rate": 1.260741462457165e-07, "loss": -0.0194, "reward": -0.04672716557979584, "reward_after_mean": -0.04672716557979584, "reward_after_std": 0.6234767735004425, "reward_before_mean": 0.2604896258562803, "reward_before_std": 0.6195750590413809, "reward_change_max": 0.0, "reward_change_mean": -0.30721679143607616, "reward_change_min": -0.5665773488581181, "reward_change_std": 0.21985585056245327, "reward_std": 0.6234767809510231, "rewards/accuracy_reward": 0.2708333395421505, "rewards/cosine_scaled_reward": -0.010343707166612148, "step": 452 }, { "clip_fraction": 0.0, "completion_length": 3104.2291870117188, "epoch": 0.5177142857142857, "grad_norm": 0.23847800493240356, "kl": 0.0392913818359375, "lambda_div_used": 0.5795375630259514, "learning_rate": 1.2503063339313356e-07, "loss": -0.0022, "reward": -0.06454518809914589, "reward_after_mean": -0.06454518809914589, "reward_after_std": 0.45274921506643295, "reward_before_mean": 0.3660498661920428, "reward_before_std": 0.36549677699804306, "reward_change_max": 0.0, "reward_change_mean": -0.4305950403213501, "reward_change_min": -0.6381234489381313, "reward_change_std": 0.2484272699803114, "reward_std": 0.45274922996759415, "rewards/accuracy_reward": 0.2916666716337204, "rewards/cosine_scaled_reward": 0.07438317034393549, "step": 453 }, { "clip_fraction": 0.0, "completion_length": 3011.5208740234375, "epoch": 0.5188571428571429, "grad_norm": 0.20881608128547668, "kl": 0.02523040771484375, "lambda_div_used": 0.6032019779086113, "learning_rate": 1.2400783294793668e-07, "loss": -0.0257, "reward": -0.24628131184726954, "reward_after_mean": -0.24628131184726954, "reward_after_std": 0.49832311272621155, "reward_before_mean": 0.03248094767332077, "reward_before_std": 0.47825742699205875, "reward_change_max": 0.0, "reward_change_mean": -0.2787622604519129, "reward_change_min": -0.5137217566370964, "reward_change_std": 0.19076244719326496, "reward_std": 0.49832311645150185, "rewards/accuracy_reward": 0.1666666716337204, "rewards/cosine_scaled_reward": -0.13418573141098022, "step": 454 }, { "clip_fraction": 0.0, "completion_length": 3486.1666870117188, "epoch": 0.52, "grad_norm": 0.20847363770008087, "kl": 0.026153564453125, "lambda_div_used": 0.558542512357235, "learning_rate": 1.2300579475997657e-07, "loss": 0.0315, "reward": -0.45310346968472004, "reward_after_mean": -0.45310346968472004, "reward_after_std": 0.32040359266102314, "reward_before_mean": -0.1906872931867838, "reward_before_std": 0.2643335796892643, "reward_change_max": 0.0, "reward_change_mean": -0.26241618394851685, "reward_change_min": -0.444510105997324, "reward_change_std": 0.15640022791922092, "reward_std": 0.32040360011160374, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.23235396784730256, "step": 455 }, { "clip_fraction": 0.0, "completion_length": 3370.1666717529297, "epoch": 0.5211428571428571, "grad_norm": 0.1887054145336151, "kl": 0.029754638671875, "lambda_div_used": 0.5725560337305069, "learning_rate": 1.220245676671809e-07, "loss": 0.0095, "reward": -0.3975539207458496, "reward_after_mean": -0.3975539207458496, "reward_after_std": 0.4078775066882372, "reward_before_mean": -0.14102032780647278, "reward_before_std": 0.32593305595219135, "reward_change_max": 0.0, "reward_change_mean": -0.2565336097031832, "reward_change_min": -0.3765680603682995, "reward_change_std": 0.13739658519625664, "reward_std": 0.4078775178641081, "rewards/accuracy_reward": 0.0416666679084301, "rewards/cosine_scaled_reward": -0.18268700037151575, "step": 456 }, { "clip_fraction": 0.0, "completion_length": 3263.500015258789, "epoch": 0.5222857142857142, "grad_norm": 0.20514518022537231, "kl": 0.0359649658203125, "lambda_div_used": 0.5794422850012779, "learning_rate": 1.2106419949317388e-07, "loss": -0.0003, "reward": -0.1368904449045658, "reward_after_mean": -0.1368904449045658, "reward_after_std": 0.45263898745179176, "reward_before_mean": 0.2523583434522152, "reward_before_std": 0.36524798441678286, "reward_change_max": 0.0, "reward_change_mean": -0.3892487920820713, "reward_change_min": -0.6007460914552212, "reward_change_std": 0.22958804108202457, "reward_std": 0.45263900980353355, "rewards/accuracy_reward": 0.2500000037252903, "rewards/cosine_scaled_reward": 0.0023583266884088516, "step": 457 }, { "clip_fraction": 0.0, "completion_length": 3182.791717529297, "epoch": 0.5234285714285715, "grad_norm": 0.2467018961906433, "kl": 0.04339599609375, "lambda_div_used": 0.613545723259449, "learning_rate": 1.2012473704494537e-07, "loss": 0.0575, "reward": -0.16568617150187492, "reward_after_mean": -0.16568617150187492, "reward_after_std": 0.5663280598819256, "reward_before_mean": 0.12488517072051764, "reward_before_std": 0.5224519474431872, "reward_change_max": 0.0, "reward_change_mean": -0.2905713580548763, "reward_change_min": -0.4896518960595131, "reward_change_std": 0.18202228285372257, "reward_std": 0.5663280598819256, "rewards/accuracy_reward": 0.1875000074505806, "rewards/cosine_scaled_reward": -0.0626148316077888, "step": 458 }, { "clip_fraction": 0.0, "completion_length": 3013.6875610351562, "epoch": 0.5245714285714286, "grad_norm": 0.5051586627960205, "kl": 0.038604736328125, "lambda_div_used": 0.6498681828379631, "learning_rate": 1.1920622611056974e-07, "loss": 0.1196, "reward": 0.16017166152596474, "reward_after_mean": 0.16017166152596474, "reward_after_std": 0.68125244602561, "reward_before_mean": 0.5180233418941498, "reward_before_std": 0.7045418079942465, "reward_change_max": 0.0, "reward_change_mean": -0.35785166546702385, "reward_change_min": -0.60487399995327, "reward_change_std": 0.25535210222005844, "reward_std": 0.6812524646520615, "rewards/accuracy_reward": 0.3958333432674408, "rewards/cosine_scaled_reward": 0.12218999862670898, "step": 459 }, { "clip_fraction": 0.0, "completion_length": 3470.7083435058594, "epoch": 0.5257142857142857, "grad_norm": 0.18247054517269135, "kl": 0.02564239501953125, "lambda_div_used": 0.6273729652166367, "learning_rate": 1.1830871145697412e-07, "loss": 0.0028, "reward": -0.19648361980216578, "reward_after_mean": -0.19648361980216578, "reward_after_std": 0.6129569914191961, "reward_before_mean": 0.05696938093751669, "reward_before_std": 0.5935165211558342, "reward_change_max": 0.0, "reward_change_mean": -0.2534530255943537, "reward_change_min": -0.50194401293993, "reward_change_std": 0.1837503593415022, "reward_std": 0.6129570361226797, "rewards/accuracy_reward": 0.16666666977107525, "rewards/cosine_scaled_reward": -0.10969728231430054, "step": 460 }, { "clip_fraction": 0.0, "completion_length": 3397.6666870117188, "epoch": 0.5268571428571428, "grad_norm": 0.1865098923444748, "kl": 0.0252227783203125, "lambda_div_used": 0.5828712657094002, "learning_rate": 1.1743223682775649e-07, "loss": 0.0154, "reward": -0.1992062833160162, "reward_after_mean": -0.1992062833160162, "reward_after_std": 0.4864924233406782, "reward_before_mean": 0.15942218992859125, "reward_before_std": 0.3801368661224842, "reward_change_max": 0.0, "reward_change_mean": -0.35862845927476883, "reward_change_min": -0.5388706848025322, "reward_change_std": 0.20263243932276964, "reward_std": 0.48649243265390396, "rewards/accuracy_reward": 0.20833333395421505, "rewards/cosine_scaled_reward": -0.048911163583397865, "step": 461 }, { "clip_fraction": 0.0, "completion_length": 3176.729217529297, "epoch": 0.528, "grad_norm": 0.20984020829200745, "kl": 0.03948974609375, "lambda_div_used": 0.5540393367409706, "learning_rate": 1.1657684494105386e-07, "loss": 0.0177, "reward": -0.5000595562160015, "reward_after_mean": -0.5000595562160015, "reward_after_std": 0.314710795879364, "reward_before_mean": -0.26233734749257565, "reward_before_std": 0.2401137463748455, "reward_change_max": 0.0, "reward_change_mean": -0.2377222292125225, "reward_change_min": -0.3677747920155525, "reward_change_std": 0.13004880584776402, "reward_std": 0.314710795879364, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.2831706702709198, "step": 462 }, { "clip_fraction": 0.0, "completion_length": 3344.0000610351562, "epoch": 0.5291428571428571, "grad_norm": 0.16413526237010956, "kl": 0.022705078125, "lambda_div_used": 0.5810666009783745, "learning_rate": 1.1574257748745986e-07, "loss": 0.0375, "reward": -0.3495323173701763, "reward_after_mean": -0.3495323173701763, "reward_after_std": 0.4335639514029026, "reward_before_mean": -0.07379204468452372, "reward_before_std": 0.3700758796185255, "reward_change_max": 0.0, "reward_change_mean": -0.27574028447270393, "reward_change_min": -0.4478879738599062, "reward_change_std": 0.1588114146143198, "reward_std": 0.43356395699083805, "rewards/accuracy_reward": 0.06250000186264515, "rewards/cosine_scaled_reward": -0.13629204593598843, "step": 463 }, { "clip_fraction": 0.0, "completion_length": 2584.750030517578, "epoch": 0.5302857142857142, "grad_norm": 0.28600558638572693, "kl": 0.12506866455078125, "lambda_div_used": 0.6016320735216141, "learning_rate": 1.1492947512799328e-07, "loss": 0.0728, "reward": 0.032141142059117556, "reward_after_mean": 0.032141142059117556, "reward_after_std": 0.61183662712574, "reward_before_mean": 0.4834227468818426, "reward_before_std": 0.46681745909154415, "reward_change_max": 0.0, "reward_change_mean": -0.45128162764012814, "reward_change_min": -0.6523922830820084, "reward_change_std": 0.24739227071404457, "reward_std": 0.6118366308510303, "rewards/accuracy_reward": 0.3750000037252903, "rewards/cosine_scaled_reward": 0.1084227726678364, "step": 464 }, { "clip_fraction": 0.0, "completion_length": 3468.4375610351562, "epoch": 0.5314285714285715, "grad_norm": 0.19893775880336761, "kl": 0.02550506591796875, "lambda_div_used": 0.6144387051463127, "learning_rate": 1.1413757749211602e-07, "loss": -0.0085, "reward": -0.27389412466436625, "reward_after_mean": -0.27389412466436625, "reward_after_std": 0.5878547765314579, "reward_before_mean": -0.028613800182938576, "reward_before_std": 0.5247924141585827, "reward_change_max": 0.0, "reward_change_mean": -0.2452803123742342, "reward_change_min": -0.4073391519486904, "reward_change_std": 0.1469843741506338, "reward_std": 0.5878547914326191, "rewards/accuracy_reward": 0.10416666977107525, "rewards/cosine_scaled_reward": -0.13278047274798155, "step": 465 }, { "clip_fraction": 0.0, "completion_length": 3319.562530517578, "epoch": 0.5325714285714286, "grad_norm": 0.204886332154274, "kl": 0.02939605712890625, "lambda_div_used": 0.5842070356011391, "learning_rate": 1.1336692317580158e-07, "loss": 0.0132, "reward": -0.2191153857856989, "reward_after_mean": -0.2191153857856989, "reward_after_std": 0.4830444473773241, "reward_before_mean": 0.12117120623588562, "reward_before_std": 0.3827533610165119, "reward_change_max": 0.0, "reward_change_mean": -0.3402865808457136, "reward_change_min": -0.5378262177109718, "reward_change_std": 0.19137921649962664, "reward_std": 0.4830444473773241, "rewards/accuracy_reward": 0.18750000186264515, "rewards/cosine_scaled_reward": -0.06632879748940468, "step": 466 }, { "clip_fraction": 0.0, "completion_length": 3459.2083435058594, "epoch": 0.5337142857142857, "grad_norm": 0.17942553758621216, "kl": 0.0366973876953125, "lambda_div_used": 0.567595362663269, "learning_rate": 1.1261754973965422e-07, "loss": 0.0185, "reward": -0.3257990628480911, "reward_after_mean": -0.3257990628480911, "reward_after_std": 0.3719336576759815, "reward_before_mean": -0.02825593249872327, "reward_before_std": 0.3043947611004114, "reward_change_max": 0.0, "reward_change_mean": -0.29754312708973885, "reward_change_min": -0.4366040602326393, "reward_change_std": 0.16578189097344875, "reward_std": 0.37193367443978786, "rewards/accuracy_reward": 0.12500000558793545, "rewards/cosine_scaled_reward": -0.1532559497281909, "step": 467 }, { "clip_fraction": 0.0, "completion_length": 3448.750030517578, "epoch": 0.5348571428571428, "grad_norm": 0.17432425916194916, "kl": 0.036895751953125, "lambda_div_used": 0.5998829826712608, "learning_rate": 1.1188949370707787e-07, "loss": 0.0135, "reward": -0.26587339164689183, "reward_after_mean": -0.26587339164689183, "reward_after_std": 0.5106639973819256, "reward_before_mean": 0.0015569590032100677, "reward_before_std": 0.4570641480386257, "reward_change_max": 0.0, "reward_change_mean": -0.2674303464591503, "reward_change_min": -0.42981093749403954, "reward_change_std": 0.1615158338099718, "reward_std": 0.5106640048325062, "rewards/accuracy_reward": 0.1041666679084301, "rewards/cosine_scaled_reward": -0.10260970704257488, "step": 468 }, { "clip_fraction": 0.0, "completion_length": 3035.041679382324, "epoch": 0.536, "grad_norm": 0.2178213894367218, "kl": 0.052947998046875, "lambda_div_used": 0.5555099248886108, "learning_rate": 1.1118279056249653e-07, "loss": -0.0012, "reward": -0.2174333520233631, "reward_after_mean": -0.2174333520233631, "reward_after_std": 0.3612515218555927, "reward_before_mean": 0.18248699884861708, "reward_before_std": 0.25087310187518597, "reward_change_max": 0.0, "reward_change_mean": -0.39992034062743187, "reward_change_min": -0.5580457299947739, "reward_change_std": 0.21861982997506857, "reward_std": 0.3612515330314636, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.025846355594694614, "step": 469 }, { "clip_fraction": 0.0, "completion_length": 3412.666748046875, "epoch": 0.5371428571428571, "grad_norm": 0.24044789373874664, "kl": 0.0541229248046875, "lambda_div_used": 0.6206874251365662, "learning_rate": 1.1049747474962444e-07, "loss": 0.0424, "reward": -0.2336212359368801, "reward_after_mean": -0.2336212359368801, "reward_after_std": 0.5980979409068823, "reward_before_mean": 0.013237725652288646, "reward_before_std": 0.5595262181013823, "reward_change_max": 0.0, "reward_change_mean": -0.24685895442962646, "reward_change_min": -0.4456180967390537, "reward_change_std": 0.16615463700145483, "reward_std": 0.5980979446321726, "rewards/accuracy_reward": 0.1458333358168602, "rewards/cosine_scaled_reward": -0.13259560312144458, "step": 470 }, { "clip_fraction": 0.0, "completion_length": 3428.2708740234375, "epoch": 0.5382857142857143, "grad_norm": 0.2026580423116684, "kl": 0.032989501953125, "lambda_div_used": 0.5628479793667793, "learning_rate": 1.0983357966978745e-07, "loss": 0.003, "reward": -0.29726970940828323, "reward_after_mean": -0.29726970940828323, "reward_after_std": 0.3957414887845516, "reward_before_mean": 0.05144570581614971, "reward_before_std": 0.2864396497607231, "reward_change_max": 0.0, "reward_change_mean": -0.34871541522443295, "reward_change_min": -0.5421463809907436, "reward_change_std": 0.19576544873416424, "reward_std": 0.39574150927364826, "rewards/accuracy_reward": 0.1666666679084301, "rewards/cosine_scaled_reward": -0.11522097233682871, "step": 471 }, { "clip_fraction": 0.0, "completion_length": 3324.354217529297, "epoch": 0.5394285714285715, "grad_norm": 0.1560915857553482, "kl": 0.025054931640625, "lambda_div_used": 0.6134910061955452, "learning_rate": 1.0919113768029517e-07, "loss": 0.0053, "reward": -0.1338162189349532, "reward_after_mean": -0.1338162189349532, "reward_after_std": 0.5834844261407852, "reward_before_mean": 0.1670103194192052, "reward_before_std": 0.5243642488494515, "reward_change_max": 0.0, "reward_change_mean": -0.3008265271782875, "reward_change_min": -0.4708016477525234, "reward_change_std": 0.18429189268499613, "reward_std": 0.583484435454011, "rewards/accuracy_reward": 0.1875000037252903, "rewards/cosine_scaled_reward": -0.020489683840423822, "step": 472 }, { "clip_fraction": 0.0, "completion_length": 3460.5, "epoch": 0.5405714285714286, "grad_norm": 0.22393003106117249, "kl": 0.02924346923828125, "lambda_div_used": 0.5691202655434608, "learning_rate": 1.0857018009286381e-07, "loss": -0.0144, "reward": -0.27680227789096534, "reward_after_mean": -0.27680227789096534, "reward_after_std": 0.449334729462862, "reward_before_mean": 0.05735496059060097, "reward_before_std": 0.31334715941920877, "reward_change_max": 0.0, "reward_change_mean": -0.3341572377830744, "reward_change_min": -0.46352221444249153, "reward_change_std": 0.1718536615371704, "reward_std": 0.44933473132550716, "rewards/accuracy_reward": 0.1666666679084301, "rewards/cosine_scaled_reward": -0.10931172431446612, "step": 473 }, { "clip_fraction": 0.0, "completion_length": 2733.7916870117188, "epoch": 0.5417142857142857, "grad_norm": 0.1851380467414856, "kl": 0.02459716796875, "lambda_div_used": 0.5772745311260223, "learning_rate": 1.0797073717209013e-07, "loss": 0.0112, "reward": -0.044522468000650406, "reward_after_mean": -0.044522468000650406, "reward_after_std": 0.5111039038747549, "reward_before_mean": 0.4149981616064906, "reward_before_std": 0.35202833265066147, "reward_change_max": 0.0, "reward_change_mean": -0.45952061004936695, "reward_change_min": -0.6568475253880024, "reward_change_std": 0.2491096304729581, "reward_std": 0.5111039187759161, "rewards/accuracy_reward": 0.33333333395421505, "rewards/cosine_scaled_reward": 0.08166481647640467, "step": 474 }, { "clip_fraction": 0.0, "completion_length": 2899.104202270508, "epoch": 0.5428571428571428, "grad_norm": 0.21433168649673462, "kl": 0.0840606689453125, "lambda_div_used": 0.6302331387996674, "learning_rate": 1.0739283813397639e-07, "loss": 0.048, "reward": 0.0836117067374289, "reward_after_mean": 0.0836117067374289, "reward_after_std": 0.6563246976584196, "reward_before_mean": 0.48081009462475777, "reward_before_std": 0.6094130510464311, "reward_change_max": 0.0, "reward_change_mean": -0.3971984013915062, "reward_change_min": -0.6311952173709869, "reward_change_std": 0.2529603438451886, "reward_std": 0.6563247069716454, "rewards/accuracy_reward": 0.35416666977107525, "rewards/cosine_scaled_reward": 0.1266434168210253, "step": 475 }, { "clip_fraction": 0.0, "completion_length": 3296.875030517578, "epoch": 0.544, "grad_norm": 0.20906521379947662, "kl": 0.02779388427734375, "lambda_div_used": 0.6422401592135429, "learning_rate": 1.068365111445064e-07, "loss": 0.053, "reward": 0.055267444501623686, "reward_after_mean": 0.055267444501623686, "reward_after_std": 0.7311443574726582, "reward_before_mean": 0.4236351568251848, "reward_before_std": 0.6584830805659294, "reward_change_max": 0.0, "reward_change_mean": -0.3683677315711975, "reward_change_min": -0.6124436557292938, "reward_change_std": 0.23675346560776234, "reward_std": 0.7311443760991096, "rewards/accuracy_reward": 0.3333333358168602, "rewards/cosine_scaled_reward": 0.09030181914567947, "step": 476 }, { "clip_fraction": 0.0, "completion_length": 2716.187545776367, "epoch": 0.5451428571428572, "grad_norm": 0.3030186593532562, "kl": 0.0340118408203125, "lambda_div_used": 0.5970823615789413, "learning_rate": 1.063017833182728e-07, "loss": 0.0606, "reward": 0.15918040089309216, "reward_after_mean": 0.15918040089309216, "reward_after_std": 0.5777834989130497, "reward_before_mean": 0.6877403110265732, "reward_before_std": 0.448156226426363, "reward_change_max": 0.0, "reward_change_mean": -0.528559859842062, "reward_change_min": -0.7873281501233578, "reward_change_std": 0.30433651618659496, "reward_std": 0.5777835063636303, "rewards/accuracy_reward": 0.4791666753590107, "rewards/cosine_scaled_reward": 0.20857359655201435, "step": 477 }, { "clip_fraction": 0.0, "completion_length": 3086.500030517578, "epoch": 0.5462857142857143, "grad_norm": 0.21326477825641632, "kl": 0.04007720947265625, "lambda_div_used": 0.5849251300096512, "learning_rate": 1.0578868071715544e-07, "loss": 0.0509, "reward": -0.1789231952279806, "reward_after_mean": -0.1789231952279806, "reward_after_std": 0.47355420142412186, "reward_before_mean": 0.1828129006025847, "reward_before_std": 0.3896525502204895, "reward_change_max": 0.0, "reward_change_mean": -0.3617360945791006, "reward_change_min": -0.5517545081675053, "reward_change_std": 0.20859801582992077, "reward_std": 0.47355422750115395, "rewards/accuracy_reward": 0.22916666977107525, "rewards/cosine_scaled_reward": -0.046353768557310104, "step": 478 }, { "clip_fraction": 0.0, "completion_length": 3312.0834045410156, "epoch": 0.5474285714285714, "grad_norm": 0.19035623967647552, "kl": 0.0276031494140625, "lambda_div_used": 0.5380282253026962, "learning_rate": 1.0529722834905125e-07, "loss": 0.0153, "reward": -0.518091470003128, "reward_after_mean": -0.518091470003128, "reward_after_std": 0.23843752220273018, "reward_before_mean": -0.2507518660277128, "reward_before_std": 0.16703186742961407, "reward_change_max": 0.0, "reward_change_mean": -0.2673396058380604, "reward_change_min": -0.401623472571373, "reward_change_std": 0.14587157778441906, "reward_std": 0.23843752779066563, "rewards/accuracy_reward": 0.0, "rewards/cosine_scaled_reward": -0.2507518734782934, "step": 479 }, { "clip_fraction": 0.0, "completion_length": 2941.6041870117188, "epoch": 0.5485714285714286, "grad_norm": 0.22068512439727783, "kl": 0.19366455078125, "lambda_div_used": 0.6076122671365738, "learning_rate": 1.0482745016665526e-07, "loss": -0.0377, "reward": -0.22745467722415924, "reward_after_mean": -0.22745467722415924, "reward_after_std": 0.5286641996353865, "reward_before_mean": 0.048221200704574585, "reward_before_std": 0.498443353921175, "reward_change_max": 0.0, "reward_change_mean": -0.2756758965551853, "reward_change_min": -0.4709361307322979, "reward_change_std": 0.17957413662225008, "reward_std": 0.5286642275750637, "rewards/accuracy_reward": 0.14583333767950535, "rewards/cosine_scaled_reward": -0.09761211555451155, "step": 480 }, { "clip_fraction": 0.0, "completion_length": 3377.229217529297, "epoch": 0.5497142857142857, "grad_norm": 0.18752911686897278, "kl": 0.0283966064453125, "lambda_div_used": 0.5932358428835869, "learning_rate": 1.0437936906629334e-07, "loss": 0.0185, "reward": -0.30091465078294277, "reward_after_mean": -0.30091465078294277, "reward_after_std": 0.4593295529484749, "reward_before_mean": -0.033269934356212616, "reward_before_std": 0.4339473620057106, "reward_change_max": 0.0, "reward_change_mean": -0.267644714564085, "reward_change_min": -0.4936486706137657, "reward_change_std": 0.17952139861881733, "reward_std": 0.4593295734375715, "rewards/accuracy_reward": 0.1041666679084301, "rewards/cosine_scaled_reward": -0.1374366097152233, "step": 481 }, { "clip_fraction": 0.0, "completion_length": 3127.229217529297, "epoch": 0.5508571428571428, "grad_norm": 0.309451162815094, "kl": 0.0305023193359375, "lambda_div_used": 0.6217934861779213, "learning_rate": 1.0395300688680625e-07, "loss": 0.0457, "reward": 0.18194390460848808, "reward_after_mean": 0.18194390460848808, "reward_after_std": 0.6137282401323318, "reward_before_mean": 0.6483347564935684, "reward_before_std": 0.5636269617825747, "reward_change_max": 0.0, "reward_change_mean": -0.4663908574730158, "reward_change_min": -0.7397138997912407, "reward_change_std": 0.2899409309029579, "reward_std": 0.6137282513082027, "rewards/accuracy_reward": 0.4375000111758709, "rewards/cosine_scaled_reward": 0.21083474811166525, "step": 482 }, { "clip_fraction": 0.0, "completion_length": 3238.3958740234375, "epoch": 0.552, "grad_norm": 0.1755090057849884, "kl": 0.036468505859375, "lambda_div_used": 0.5799357891082764, "learning_rate": 1.0354838440848501e-07, "loss": 0.0127, "reward": -0.34550213161855936, "reward_after_mean": -0.34550213161855936, "reward_after_std": 0.4019248206168413, "reward_before_mean": -0.07407787069678307, "reward_before_std": 0.36604968179017305, "reward_change_max": 0.0, "reward_change_mean": -0.2714242674410343, "reward_change_min": -0.4708656519651413, "reward_change_std": 0.17479356285184622, "reward_std": 0.4019248355180025, "rewards/accuracy_reward": 0.1041666679084301, "rewards/cosine_scaled_reward": -0.1782445318531245, "step": 483 }, { "clip_fraction": 0.0, "completion_length": 2761.750015258789, "epoch": 0.5531428571428572, "grad_norm": 0.2134392112493515, "kl": 0.0371856689453125, "lambda_div_used": 0.5781532227993011, "learning_rate": 1.0316552135205837e-07, "loss": 0.02, "reward": -0.029964998364448547, "reward_after_mean": -0.029964998364448547, "reward_after_std": 0.4321981817483902, "reward_before_mean": 0.41219746321439743, "reward_before_std": 0.35605006478726864, "reward_change_max": 0.0, "reward_change_mean": -0.44216244481503963, "reward_change_min": -0.6350057646632195, "reward_change_std": 0.25374310091137886, "reward_std": 0.43219820596277714, "rewards/accuracy_reward": 0.3333333432674408, "rewards/cosine_scaled_reward": 0.07886410318315029, "step": 484 }, { "clip_fraction": 0.0, "completion_length": 3221.8125610351562, "epoch": 0.5542857142857143, "grad_norm": 0.2568052411079407, "kl": 0.03658294677734375, "lambda_div_used": 0.5969985648989677, "learning_rate": 1.0280443637773163e-07, "loss": 0.0464, "reward": -0.2202933905646205, "reward_after_mean": -0.2202933905646205, "reward_after_std": 0.49520318396389484, "reward_before_mean": 0.08446505106985569, "reward_before_std": 0.4468123037368059, "reward_change_max": 0.0, "reward_change_mean": -0.30475846491754055, "reward_change_min": -0.4917575791478157, "reward_change_std": 0.18758091609925032, "reward_std": 0.4952031895518303, "rewards/accuracy_reward": 0.14583333767950535, "rewards/cosine_scaled_reward": -0.061368280090391636, "step": 485 }, { "clip_fraction": 0.0, "completion_length": 2887.729179382324, "epoch": 0.5554285714285714, "grad_norm": 0.24756214022636414, "kl": 0.0624237060546875, "lambda_div_used": 0.5388600528240204, "learning_rate": 1.0246514708427701e-07, "loss": -0.014, "reward": -0.3731558360159397, "reward_after_mean": -0.3731558360159397, "reward_after_std": 0.2838398087769747, "reward_before_mean": -0.00787089392542839, "reward_before_std": 0.17069030087441206, "reward_change_max": 0.0, "reward_change_mean": -0.36528492718935013, "reward_change_min": -0.5211359187960625, "reward_change_std": 0.19405403640121222, "reward_std": 0.2838398218154907, "rewards/accuracy_reward": 0.125, "rewards/cosine_scaled_reward": -0.13287090510129929, "step": 486 }, { "clip_fraction": 0.0, "completion_length": 2605.8750534057617, "epoch": 0.5565714285714286, "grad_norm": 0.2897025942802429, "kl": 0.073577880859375, "lambda_div_used": 0.590158075094223, "learning_rate": 1.0214767000817596e-07, "loss": 0.0132, "reward": -0.10170878353528678, "reward_after_mean": -0.10170878353528678, "reward_after_std": 0.5194713845849037, "reward_before_mean": 0.27859709272161126, "reward_before_std": 0.40841816924512386, "reward_change_max": 0.0, "reward_change_mean": -0.3803058788180351, "reward_change_min": -0.5254599489271641, "reward_change_std": 0.20420703664422035, "reward_std": 0.5194714032113552, "rewards/accuracy_reward": 0.2708333395421505, "rewards/cosine_scaled_reward": 0.00776377459987998, "step": 487 }, { "clip_fraction": 0.0, "completion_length": 2796.541717529297, "epoch": 0.5577142857142857, "grad_norm": 0.2172238528728485, "kl": 0.0294036865234375, "lambda_div_used": 0.5966172888875008, "learning_rate": 1.0185202062281336e-07, "loss": -0.0073, "reward": -0.1289364038966596, "reward_after_mean": -0.1289364038966596, "reward_after_std": 0.4726745132356882, "reward_before_mean": 0.20998166128993034, "reward_before_std": 0.43987042736262083, "reward_change_max": 0.0, "reward_change_mean": -0.33891804702579975, "reward_change_min": -0.5095071531832218, "reward_change_std": 0.20220207329839468, "reward_std": 0.4726745318621397, "rewards/accuracy_reward": 0.22916667722165585, "rewards/cosine_scaled_reward": -0.01918501779437065, "step": 488 }, { "clip_fraction": 0.0, "completion_length": 3544.0, "epoch": 0.5588571428571428, "grad_norm": 0.20985524356365204, "kl": 0.0297698974609375, "lambda_div_used": 0.5526714697480202, "learning_rate": 1.0157821333772304e-07, "loss": -0.0039, "reward": -0.483198668807745, "reward_after_mean": -0.483198668807745, "reward_after_std": 0.3155835196375847, "reward_before_mean": -0.23311096336692572, "reward_before_std": 0.23472226411104202, "reward_change_max": 0.0, "reward_change_mean": -0.25008769892156124, "reward_change_min": -0.369185172021389, "reward_change_std": 0.13414029870182276, "reward_std": 0.31558352895081043, "rewards/accuracy_reward": 0.02083333395421505, "rewards/cosine_scaled_reward": -0.25394430477172136, "step": 489 }, { "clip_fraction": 0.0, "completion_length": 2594.2708892822266, "epoch": 0.56, "grad_norm": 0.21995790302753448, "kl": 0.0242462158203125, "lambda_div_used": 0.5830037221312523, "learning_rate": 1.013262614978859e-07, "loss": 0.0504, "reward": -0.11565294116735458, "reward_after_mean": -0.11565294116735458, "reward_after_std": 0.45833847112953663, "reward_before_mean": 0.27795974537730217, "reward_before_std": 0.3805103488266468, "reward_change_max": 0.0, "reward_change_mean": -0.39361269399523735, "reward_change_min": -0.5925287120044231, "reward_change_std": 0.23201382905244827, "reward_std": 0.4583384860306978, "rewards/accuracy_reward": 0.2708333358168602, "rewards/cosine_scaled_reward": 0.007126402109861374, "step": 490 }, { "clip_fraction": 0.0, "completion_length": 3157.937530517578, "epoch": 0.5611428571428572, "grad_norm": 0.19164642691612244, "kl": 0.03619384765625, "lambda_div_used": 0.6008268222212791, "learning_rate": 1.0109617738307911e-07, "loss": 0.023, "reward": 0.019691139459609985, "reward_after_mean": 0.019691139459609985, "reward_after_std": 0.5389490965753794, "reward_before_mean": 0.44501334615051746, "reward_before_std": 0.46576130110770464, "reward_change_max": 0.0, "reward_change_mean": -0.4253222309052944, "reward_change_min": -0.6597434133291245, "reward_change_std": 0.2594317300245166, "reward_std": 0.5389491245150566, "rewards/accuracy_reward": 0.3541666753590107, "rewards/cosine_scaled_reward": 0.09084668569266796, "step": 491 }, { "clip_fraction": 0.0, "completion_length": 2851.104217529297, "epoch": 0.5622857142857143, "grad_norm": 0.2606635093688965, "kl": 0.0465545654296875, "lambda_div_used": 0.5979261994361877, "learning_rate": 1.0088797220727779e-07, "loss": 0.0176, "reward": -0.24230816727504134, "reward_after_mean": -0.24230816727504134, "reward_after_std": 0.4972796719521284, "reward_before_mean": 0.03912084607873112, "reward_before_std": 0.4469980522990227, "reward_change_max": 0.0, "reward_change_mean": -0.2814290430396795, "reward_change_min": -0.4379620999097824, "reward_change_std": 0.16612732410430908, "reward_std": 0.49727969244122505, "rewards/accuracy_reward": 0.1250000037252903, "rewards/cosine_scaled_reward": -0.08587915264070034, "step": 492 }, { "clip_fraction": 0.0, "completion_length": 2895.8334045410156, "epoch": 0.5634285714285714, "grad_norm": 0.19319528341293335, "kl": 0.03064727783203125, "lambda_div_used": 0.5970403179526329, "learning_rate": 1.0070165611810855e-07, "loss": 0.0528, "reward": 0.10795958898961544, "reward_after_mean": 0.10795958898961544, "reward_after_std": 0.594191886484623, "reward_before_mean": 0.6057661911472678, "reward_before_std": 0.44818239752203226, "reward_change_max": 0.0, "reward_change_mean": -0.4978065937757492, "reward_change_min": -0.6970877684652805, "reward_change_std": 0.27646912820637226, "reward_std": 0.594191899523139, "rewards/accuracy_reward": 0.41666666977107525, "rewards/cosine_scaled_reward": 0.18909949623048306, "step": 493 }, { "clip_fraction": 0.0, "completion_length": 2822.145950317383, "epoch": 0.5645714285714286, "grad_norm": 0.21403808891773224, "kl": 0.03521728515625, "lambda_div_used": 0.6499281525611877, "learning_rate": 1.005372381963547e-07, "loss": -0.0124, "reward": 0.13442726712673903, "reward_after_mean": 0.13442726712673903, "reward_after_std": 0.7398196011781693, "reward_before_mean": 0.5234599430114031, "reward_before_std": 0.70114840939641, "reward_change_max": 0.0, "reward_change_mean": -0.38903267681598663, "reward_change_min": -0.6715483777225018, "reward_change_std": 0.263275028206408, "reward_std": 0.7398196253925562, "rewards/accuracy_reward": 0.39583333767950535, "rewards/cosine_scaled_reward": 0.12762661091983318, "step": 494 }, { "clip_fraction": 0.0, "completion_length": 3231.8958587646484, "epoch": 0.5657142857142857, "grad_norm": 0.19664397835731506, "kl": 0.029449462890625, "lambda_div_used": 0.6409478113055229, "learning_rate": 1.0039472645551372e-07, "loss": -0.013, "reward": 0.07278990373015404, "reward_after_mean": 0.07278990373015404, "reward_after_std": 0.6243610084056854, "reward_before_mean": 0.4131181836128235, "reward_before_std": 0.6622960511595011, "reward_change_max": 0.0, "reward_change_mean": -0.34032829850912094, "reward_change_min": -0.5942648649215698, "reward_change_std": 0.2463043536990881, "reward_std": 0.6243610270321369, "rewards/accuracy_reward": 0.3125000074505806, "rewards/cosine_scaled_reward": 0.10061818920075893, "step": 495 }, { "clip_fraction": 0.0, "completion_length": 2522.6041717529297, "epoch": 0.5668571428571428, "grad_norm": 0.27453672885894775, "kl": 0.05194854736328125, "lambda_div_used": 0.5593710169196129, "learning_rate": 1.002741278414069e-07, "loss": -0.0122, "reward": -0.11981088295578957, "reward_after_mean": -0.11981088295578957, "reward_after_std": 0.43237836100161076, "reward_before_mean": 0.33947549015283585, "reward_before_std": 0.26912583224475384, "reward_change_max": 0.0, "reward_change_mean": -0.45928638987243176, "reward_change_min": -0.6750613376498222, "reward_change_std": 0.24631350580602884, "reward_std": 0.4323783814907074, "rewards/accuracy_reward": 0.2916666679084301, "rewards/cosine_scaled_reward": 0.047808803617954254, "step": 496 }, { "clip_fraction": 0.0, "completion_length": 2998.833366394043, "epoch": 0.568, "grad_norm": 0.20906421542167664, "kl": 0.0264129638671875, "lambda_div_used": 0.5815169364213943, "learning_rate": 1.0017544823184055e-07, "loss": -0.0008, "reward": 0.027008600533008575, "reward_after_mean": 0.027008600533008575, "reward_after_std": 0.5124967209994793, "reward_before_mean": 0.5082651861011982, "reward_before_std": 0.3809623853303492, "reward_change_max": 0.0, "reward_change_mean": -0.4812565799802542, "reward_change_min": -0.6963069848716259, "reward_change_std": 0.2742460994049907, "reward_std": 0.512496730312705, "rewards/accuracy_reward": 0.3958333358168602, "rewards/cosine_scaled_reward": 0.11243184097111225, "step": 497 }, { "clip_fraction": 0.0, "completion_length": 3297.312530517578, "epoch": 0.5691428571428572, "grad_norm": 0.20767350494861603, "kl": 0.0320587158203125, "lambda_div_used": 0.5861091017723083, "learning_rate": 1.0009869243631952e-07, "loss": 0.0121, "reward": -0.18442361429333687, "reward_after_mean": -0.18442361429333687, "reward_after_std": 0.4828739557415247, "reward_before_mean": 0.17697326093912125, "reward_before_std": 0.39562609419226646, "reward_change_max": 0.0, "reward_change_mean": -0.36139689199626446, "reward_change_min": -0.5951493978500366, "reward_change_std": 0.21696829982101917, "reward_std": 0.4828739669173956, "rewards/accuracy_reward": 0.2291666679084301, "rewards/cosine_scaled_reward": -0.05219341011252254, "step": 498 }, { "clip_fraction": 0.0, "completion_length": 3079.7083435058594, "epoch": 0.5702857142857143, "grad_norm": 0.18754175305366516, "kl": 0.0654449462890625, "lambda_div_used": 0.6014531627297401, "learning_rate": 1.000438641958131e-07, "loss": 0.0132, "reward": -0.06358168926090002, "reward_after_mean": -0.06358168926090002, "reward_after_std": 0.5510079935193062, "reward_before_mean": 0.31686626374721527, "reward_before_std": 0.4676083065569401, "reward_change_max": 0.0, "reward_change_mean": -0.380447955802083, "reward_change_min": -0.5952403843402863, "reward_change_std": 0.2253542598336935, "reward_std": 0.5510080195963383, "rewards/accuracy_reward": 0.27083333767950535, "rewards/cosine_scaled_reward": 0.046032913494855165, "step": 499 }, { "clip_fraction": 0.0, "completion_length": 3479.625030517578, "epoch": 0.5714285714285714, "grad_norm": 0.20164164900779724, "kl": 0.0302886962890625, "lambda_div_used": 0.6078668311238289, "learning_rate": 1.0001096618257236e-07, "loss": 0.0208, "reward": -0.14625994116067886, "reward_after_mean": -0.14625994116067886, "reward_after_std": 0.5143183693289757, "reward_before_mean": 0.1679236814379692, "reward_before_std": 0.5014720680192113, "reward_change_max": 0.0, "reward_change_mean": -0.31418363377451897, "reward_change_min": -0.5387452095746994, "reward_change_std": 0.21335808746516705, "reward_std": 0.5143184047192335, "rewards/accuracy_reward": 0.2083333358168602, "rewards/cosine_scaled_reward": -0.04040965251624584, "step": 500 }, { "epoch": 0.5714285714285714, "step": 500, "total_flos": 0.0, "train_loss": 0.019718800745411137, "train_runtime": 79297.6557, "train_samples_per_second": 0.303, "train_steps_per_second": 0.006 } ], "logging_steps": 1, "max_steps": 500, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 6, "trial_name": null, "trial_params": null }