Upload SLMoE 100% checkpoint at 59,999,518,720 tokens

README.md

@@ -19,9 +19,9 @@ tags:
 - trust-remote-code
 ---
 
-# SLMoE Test - 95%
+# SLMoE Test - 100%
 
-This is the **95% checkpoint** of an experimental sequence-routed base
+This is the **100% checkpoint** of an experimental sequence-routed base
 language model. It is not instruction tuned.
 
 ## Architecture
@@ -48,8 +48,8 @@ and loss on its routing prefix is masked to prevent future-token leakage.
 
 | Field | Value |
 |---|---:|
-| Progress | 95% |
-| Tokens seen | 57,000,591,360 |
+| Progress | 100% |
+| Tokens seen | 59,999,518,720 |
 | Target tokens | 60,000,000,000 |
 | Hardware | 4 x NVIDIA H200 |
 | Optimizer | Fused AdamW |

checkpoint_metadata.json

@@ -46,10 +46,10 @@
     "model_type": "slmoe",
     "output_attentions": false
   },
-  "training_percent": 95,
-  "step": 27180,
+  "training_percent": 100,
+  "step": 28610,
   "total_steps": 28610,
-  "tokens_seen": 57000591360,
+  "tokens_seen": 59999518720,
   "target_tokens": 60000000000,
   "trained_token_target": 59999518720,
   "tokens_per_step": 2097152,
@@ -73,12 +73,12 @@
     "npset2": "222b911cd9119624dd65ab084d69542f6e04631e"
   },
   "source_tokens": {
-    "fineweb_hq": 16320036864,
-    "fineweb_edu": 16229859328,
-    "dclm": 9059696640,
-    "cosmopedia_v2": 7050625024,
-    "finemath": 6689914880,
-    "npset2": 1650458624
+    "fineweb_hq": 16919822336,
+    "fineweb_edu": 16949182464,
+    "dclm": 9330229248,
+    "cosmopedia_v2": 7499415552,
+    "finemath": 7470055424,
+    "npset2": 1830813696
   },
-  "elapsed_seconds": 15171.971340417862
+  "elapsed_seconds": 15931.297726869583
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:18cdb956bdb02aaa03e33617385e842ae555bf7feaab8a6df854cf7791268f85
+oid sha256:43d8448094e6bd0710ad287107bb8ba64407358fe191895eafc421f89b7f7262
 size 110196488

training_metrics.jsonl

@@ -2717,3 +2717,146 @@
 {"step": 27160, "total_steps": 28610, "tokens": 56958648320, "source": "cosmopedia_v2", "phase": 5, "loss": 2.7658964097499847, "ce_loss": 2.753924936056137, "logit_z_loss": 27.595478057861328, "router_aux_loss": 1.1256768256425858, "router_z_loss": 0.7147043198347092, "routing_entropy_normalized": 0.7230427233392426, "learning_rate": 0.0007691056266308628, "grad_norm": 1.1873530149459839, "tokens_per_second": 3942826.1392121166, "eta_seconds": 771.2413108348846, "data_wait_seconds": 2.3990869522094727e-05}
 {"step": 27170, "total_steps": 28610, "tokens": 56979619840, "source": "finemath", "phase": 5, "loss": 2.1898454129695892, "ce_loss": 2.1781243085861206, "logit_z_loss": 35.876676082611084, "router_aux_loss": 1.1464245170354843, "router_z_loss": 0.25689063780009747, "routing_entropy_normalized": 0.6990924799985128, "learning_rate": 0.0007595329588153221, "grad_norm": 0.9159744381904602, "tokens_per_second": 4046207.15202699, "eta_seconds": 746.3530082702637, "data_wait_seconds": 2.0205974578857422e-05}
 {"step": 27180, "total_steps": 28610, "tokens": 57000591360, "source": "cosmopedia_v2", "phase": 5, "loss": 2.744966447353363, "ce_loss": 2.732774317264557, "logit_z_loss": 27.7472026348114, "router_aux_loss": 1.1514736115932465, "router_z_loss": 0.6773731112480164, "routing_entropy_normalized": 0.66993255428104, "learning_rate": 0.0007500000000000003, "grad_norm": 3.3361501693725586, "tokens_per_second": 4152975.482551904, "eta_seconds": 722.1153538227081, "data_wait_seconds": 2.0563602447509766e-05}
+{"step": 27190, "total_steps": 28610, "tokens": 57021562880, "source": "fineweb_edu", "phase": 5, "loss": 3.359998971223831, "ce_loss": 3.3480939865112305, "logit_z_loss": 24.516781330108643, "router_aux_loss": 1.1459475606679916, "router_z_loss": 0.4454890415072441, "routing_entropy_normalized": 0.7108187501580212, "learning_rate": 0.0007405072614085462, "grad_norm": 0.45561233162879944, "tokens_per_second": 4610843.146975151, "eta_seconds": 645.8592810630798, "data_wait_seconds": 1.8417835235595703e-05}
+{"step": 27200, "total_steps": 28610, "tokens": 57042534400, "source": "cosmopedia_v2", "phase": 5, "loss": 2.73199862241745, "ce_loss": 2.7198681831359863, "logit_z_loss": 27.525178909301758, "router_aux_loss": 1.144388496875763, "router_z_loss": 0.6865886449813843, "routing_entropy_normalized": 0.7208260226315351, "learning_rate": 0.0007310552521077218, "grad_norm": 0.8272308707237244, "tokens_per_second": 3772949.6790551734, "eta_seconds": 783.7327744960785, "data_wait_seconds": 2.008676528930664e-05}
+{"step": 27210, "total_steps": 28610, "tokens": 57063505920, "source": "npset2", "phase": 5, "loss": 1.6135769188404083, "ce_loss": 1.6016628444194794, "logit_z_loss": 40.03337001800537, "router_aux_loss": 1.143881395459175, "router_z_loss": 0.4752824157476425, "routing_entropy_normalized": 0.6523975004399571, "learning_rate": 0.0007216444789801009, "grad_norm": 0.7584613561630249, "tokens_per_second": 3970799.9516286203, "eta_seconds": 739.4008350372314, "data_wait_seconds": 1.817941665649414e-05}
+{"step": 27220, "total_steps": 28610, "tokens": 57084477440, "source": "fineweb_edu", "phase": 5, "loss": 3.380850166082382, "ce_loss": 3.3689653277397156, "logit_z_loss": 24.89562439918518, "router_aux_loss": 1.1439179182052612, "router_z_loss": 0.44564804807305336, "routing_entropy_normalized": 0.7003654366896488, "learning_rate": 0.0007122754466968858, "grad_norm": 0.47999805212020874, "tokens_per_second": 4251250.409344301, "eta_seconds": 685.6903262138367, "data_wait_seconds": 2.1249055862426758e-05}
+{"step": 27230, "total_steps": 28610, "tokens": 57105448960, "source": "dclm", "phase": 5, "loss": 3.563316822052002, "ce_loss": 3.551588863134384, "logit_z_loss": 25.18594217300415, "router_aux_loss": 1.1399169266223907, "router_z_loss": 0.3287147842347622, "routing_entropy_normalized": 0.7041120360176503, "learning_rate": 0.0007029486576908444, "grad_norm": 0.7160404324531555, "tokens_per_second": 3840182.775315622, "eta_seconds": 753.6281290054321, "data_wait_seconds": 1.677870750427246e-05}
+{"step": 27240, "total_steps": 28610, "tokens": 57126420480, "source": "finemath", "phase": 5, "loss": 2.2588150948286057, "ce_loss": 2.247038945555687, "logit_z_loss": 35.96598148345947, "router_aux_loss": 1.1464468389749527, "router_z_loss": 0.31171043775975704, "routing_entropy_normalized": 0.6681863720243265, "learning_rate": 0.0006936646121293654, "grad_norm": 0.7615252137184143, "tokens_per_second": 3818417.5935541485, "eta_seconds": 752.4316473007202, "data_wait_seconds": 1.9043684005737305e-05}
+{"step": 27250, "total_steps": 28610, "tokens": 57147392000, "source": "dclm", "phase": 5, "loss": 3.519827753305435, "ce_loss": 3.5080226957798004, "logit_z_loss": 25.702720642089844, "router_aux_loss": 1.1436737477779388, "router_z_loss": 0.36832766607403755, "routing_entropy_normalized": 0.6813647329517901, "learning_rate": 0.000684423807887638, "grad_norm": 1.0633331537246704, "tokens_per_second": 4141759.7990278928, "eta_seconds": 688.626781463623, "data_wait_seconds": 2.0205974578857422e-05}
+{"step": 27260, "total_steps": 28610, "tokens": 57168363520, "source": "cosmopedia_v2", "phase": 5, "loss": 2.710816890001297, "ce_loss": 2.69890233874321, "logit_z_loss": 29.35981559753418, "router_aux_loss": 1.1305030137300491, "router_z_loss": 0.6095302924513817, "routing_entropy_normalized": 0.7062633925879139, "learning_rate": 0.0006752267405219478, "grad_norm": 1.5235810279846191, "tokens_per_second": 4438035.205420683, "eta_seconds": 637.9298651218414, "data_wait_seconds": 2.1189451217651367e-05}
+{"step": 27270, "total_steps": 28610, "tokens": 57189335040, "source": "finemath", "phase": 5, "loss": 2.2848664224147797, "ce_loss": 2.2729388773441315, "logit_z_loss": 36.83523941040039, "router_aux_loss": 1.1691796481609344, "router_z_loss": 0.23578236997127533, "routing_entropy_normalized": 0.6808883757728483, "learning_rate": 0.0006660739032431065, "grad_norm": 1.8410810232162476, "tokens_per_second": 3651096.287434704, "eta_seconds": 769.6821608543396, "data_wait_seconds": 2.0563602447509766e-05}
+{"step": 27280, "total_steps": 28610, "tokens": 57210306560, "source": "cosmopedia_v2", "phase": 5, "loss": 2.697314113378525, "ce_loss": 2.684768259525299, "logit_z_loss": 28.13218903541565, "router_aux_loss": 1.1947495341300964, "router_z_loss": 0.598373532295227, "routing_entropy_normalized": 0.6577776415370906, "learning_rate": 0.00065696578689, "grad_norm": 1.1601126194000244, "tokens_per_second": 4193387.0005271714, "eta_seconds": 665.1454205513, "data_wait_seconds": 2.2739171981811523e-05}
+{"step": 27290, "total_steps": 28610, "tokens": 57231278080, "source": "fineweb_edu", "phase": 5, "loss": 3.3681235313415527, "ce_loss": 3.355970174074173, "logit_z_loss": 25.04009485244751, "router_aux_loss": 1.1735046356916428, "router_z_loss": 0.41834935918450356, "routing_entropy_normalized": 0.6899626958547367, "learning_rate": 0.0006479028799032665, "grad_norm": 1.1514455080032349, "tokens_per_second": 4563739.935056115, "eta_seconds": 606.5728282928467, "data_wait_seconds": 2.1725893020629883e-05}
+{"step": 27300, "total_steps": 28610, "tokens": 57252249600, "source": "cosmopedia_v2", "phase": 5, "loss": 2.697975993156433, "ce_loss": 2.685825824737549, "logit_z_loss": 28.32142186164856, "router_aux_loss": 1.152631938457489, "router_z_loss": 0.6238861307501793, "routing_entropy_normalized": 0.7051103679233057, "learning_rate": 0.0006388856682991034, "grad_norm": 1.2602440118789673, "tokens_per_second": 3905760.372682683, "eta_seconds": 703.3890607357025, "data_wait_seconds": 2.0176172256469727e-05}
+{"step": 27310, "total_steps": 28610, "tokens": 57273221120, "source": "npset2", "phase": 5, "loss": 1.5471699982881546, "ce_loss": 1.535287082195282, "logit_z_loss": 40.03988027572632, "router_aux_loss": 1.1413107365369797, "router_z_loss": 0.4698137529194355, "routing_entropy_normalized": 0.6981074530078091, "learning_rate": 0.000629914635643203, "grad_norm": 0.8176607489585876, "tokens_per_second": 3504955.8867586073, "eta_seconds": 777.8407740592957, "data_wait_seconds": 2.2232532501220703e-05}
+{"step": 27320, "total_steps": 28610, "tokens": 57294192640, "source": "fineweb_edu", "phase": 5, "loss": 3.3347651660442352, "ce_loss": 3.323015868663788, "logit_z_loss": 24.890592575073242, "router_aux_loss": 1.1315443217754364, "router_z_loss": 0.4339235797524452, "routing_entropy_normalized": 0.7069728879197142, "learning_rate": 0.000620990263024821, "grad_norm": 1.1690829992294312, "tokens_per_second": 4539463.978479164, "eta_seconds": 595.9571642875671, "data_wait_seconds": 2.014636993408203e-05}
+{"step": 27330, "total_steps": 28610, "tokens": 57315164160, "source": "dclm", "phase": 5, "loss": 3.4685001373291016, "ce_loss": 3.456776410341263, "logit_z_loss": 25.152488231658936, "router_aux_loss": 1.1367897987365723, "router_z_loss": 0.3557678982615471, "routing_entropy_normalized": 0.668205205625685, "learning_rate": 0.000612113029030977, "grad_norm": 1.41571843624115, "tokens_per_second": 4204147.246170723, "eta_seconds": 638.5015563964843, "data_wait_seconds": 1.9818544387817383e-05}
+{"step": 27340, "total_steps": 28610, "tokens": 57336135680, "source": "finemath", "phase": 5, "loss": 2.173094943165779, "ce_loss": 2.1611558198928833, "logit_z_loss": 36.4738974571228, "router_aux_loss": 1.166327714920044, "router_z_loss": 0.275822164490819, "routing_entropy_normalized": 0.6623565638146336, "learning_rate": 0.0006032834097207889, "grad_norm": 0.47295820713043213, "tokens_per_second": 3956775.795425879, "eta_seconds": 673.1195240020752, "data_wait_seconds": 2.136826515197754e-05}
+{"step": 27350, "total_steps": 28610, "tokens": 57357107200, "source": "dclm", "phase": 5, "loss": 3.512218713760376, "ce_loss": 3.5002668499946594, "logit_z_loss": 25.425736665725708, "router_aux_loss": 1.1613765507936478, "router_z_loss": 0.33813730627298355, "routing_entropy_normalized": 0.6550750432951279, "learning_rate": 0.0005945018785999439, "grad_norm": 0.3904390335083008, "tokens_per_second": 4222553.999950555, "eta_seconds": 625.7851338386536, "data_wait_seconds": 2.199411392211914e-05}
+{"step": 27360, "total_steps": 28610, "tokens": 57378078720, "source": "cosmopedia_v2", "phase": 5, "loss": 2.7119091153144836, "ce_loss": 2.699716478586197, "logit_z_loss": 27.793972492218018, "router_aux_loss": 1.1521059572696686, "router_z_loss": 0.6715151593089104, "routing_entropy_normalized": 0.6942042894758622, "learning_rate": 0.0005857689065953056, "grad_norm": 2.734346866607666, "tokens_per_second": 4051173.6775461975, "eta_seconds": 647.0816135406494, "data_wait_seconds": 1.9729137420654297e-05}
+{"step": 27370, "total_steps": 28610, "tokens": 57399050240, "source": "finemath", "phase": 5, "loss": 2.120582640171051, "ce_loss": 2.1086553931236267, "logit_z_loss": 36.30908918380737, "router_aux_loss": 1.1621994525194168, "router_z_loss": 0.30525220930576324, "routing_entropy_normalized": 0.6844714568947154, "learning_rate": 0.0005770849620296591, "grad_norm": 0.7781955003738403, "tokens_per_second": 3567065.8765166714, "eta_seconds": 729.0217142105103, "data_wait_seconds": 2.276897430419922e-05}
+{"step": 27380, "total_steps": 28610, "tokens": 57420021760, "source": "cosmopedia_v2", "phase": 5, "loss": 2.701281398534775, "ce_loss": 2.6891345381736755, "logit_z_loss": 28.288653135299683, "router_aux_loss": 1.145484060049057, "router_z_loss": 0.6920041218400002, "routing_entropy_normalized": 0.6485221301872393, "learning_rate": 0.0005684505105965974, "grad_norm": 1.1340293884277344, "tokens_per_second": 4191959.111678167, "eta_seconds": 615.3440172672272, "data_wait_seconds": 2.4318695068359375e-05}
+{"step": 27390, "total_steps": 28610, "tokens": 57440993280, "source": "fineweb_edu", "phase": 5, "loss": 3.3724576234817505, "ce_loss": 3.3607043623924255, "logit_z_loss": 24.385291576385498, "router_aux_loss": 1.13286592066288, "router_z_loss": 0.42461635917425156, "routing_entropy_normalized": 0.7101786999377885, "learning_rate": 0.0005598660153355462, "grad_norm": 0.7322582602500916, "tokens_per_second": 4155169.577738428, "eta_seconds": 615.7451319694519, "data_wait_seconds": 2.1010637283325195e-05}
+{"step": 27400, "total_steps": 28610, "tokens": 57461964800, "source": "cosmopedia_v2", "phase": 5, "loss": 3.066123515367508, "ce_loss": 3.0538840293884277, "logit_z_loss": 26.53302240371704, "router_aux_loss": 1.1574629098176956, "router_z_loss": 0.6648423671722412, "routing_entropy_normalized": 0.6958644045276065, "learning_rate": 0.0005513319366069342, "grad_norm": 2.1996982097625732, "tokens_per_second": 3933601.4657241865, "eta_seconds": 645.096851348877, "data_wait_seconds": 2.2351741790771484e-05}
+{"step": 27410, "total_steps": 28610, "tokens": 57482936320, "source": "npset2", "phase": 5, "loss": 1.7342600226402283, "ce_loss": 1.7217050939798355, "logit_z_loss": 36.7751145362854, "router_aux_loss": 1.2107395231723785, "router_z_loss": 0.44751112535595894, "routing_entropy_normalized": 0.688461323638076, "learning_rate": 0.0005428487320675038, "grad_norm": 0.9691622257232666, "tokens_per_second": 3902544.8761108173, "eta_seconds": 644.8567485809326, "data_wait_seconds": 2.104043960571289e-05}
+{"step": 27420, "total_steps": 28610, "tokens": 57503907840, "source": "fineweb_edu", "phase": 5, "loss": 3.5529070794582367, "ce_loss": 3.540081173181534, "logit_z_loss": 23.973432302474976, "router_aux_loss": 1.2377067357301712, "router_z_loss": 0.4488038830459118, "routing_entropy_normalized": 0.663651910028144, "learning_rate": 0.0005344168566457695, "grad_norm": 0.4208899438381195, "tokens_per_second": 4351275.951286408, "eta_seconds": 573.5354199409485, "data_wait_seconds": 1.9997358322143555e-05}
+{"step": 27430, "total_steps": 28610, "tokens": 57524879360, "source": "dclm", "phase": 5, "loss": 3.6620669662952423, "ce_loss": 3.649735450744629, "logit_z_loss": 24.382521152496338, "router_aux_loss": 1.1981806010007858, "router_z_loss": 0.34979190677404404, "routing_entropy_normalized": 0.6724229517281588, "learning_rate": 0.0005260367625176202, "grad_norm": 0.61320561170578, "tokens_per_second": 3859270.369658558, "eta_seconds": 641.219485282898, "data_wait_seconds": 1.767277717590332e-05}
+{"step": 27440, "total_steps": 28610, "tokens": 57545850880, "source": "finemath", "phase": 5, "loss": 2.1695883721113205, "ce_loss": 2.1575977951288223, "logit_z_loss": 36.49573564529419, "router_aux_loss": 1.1722975671291351, "router_z_loss": 0.2676702532917261, "routing_entropy_normalized": 0.7403342219565815, "learning_rate": 0.0005177088990820725, "grad_norm": 0.5980548858642578, "tokens_per_second": 4184069.8326085336, "eta_seconds": 586.4309005737305, "data_wait_seconds": 1.946091651916504e-05}
+{"step": 27450, "total_steps": 28610, "tokens": 57566822400, "source": "dclm", "phase": 5, "loss": 3.6363372802734375, "ce_loss": 3.62434184551239, "logit_z_loss": 24.358548164367676, "router_aux_loss": 1.164070188999176, "router_z_loss": 0.35477619618177414, "routing_entropy_normalized": 0.7508924626149794, "learning_rate": 0.0005094337129371681, "grad_norm": 0.6511759161949158, "tokens_per_second": 4112515.9669025447, "eta_seconds": 591.5348024368286, "data_wait_seconds": 2.440810203552246e-05}
+{"step": 27460, "total_steps": 28610, "tokens": 57587793920, "source": "cosmopedia_v2", "phase": 5, "loss": 2.8173894584178925, "ce_loss": 2.805150091648102, "logit_z_loss": 27.46114730834961, "router_aux_loss": 1.1602921336889267, "router_z_loss": 0.6364258527755737, "routing_entropy_normalized": 0.7112096871838839, "learning_rate": 0.0005012116478560263, "grad_norm": 1.9540283679962158, "tokens_per_second": 3596758.201289896, "eta_seconds": 670.5273652076721, "data_wait_seconds": 2.3305416107177734e-05}
+{"step": 27470, "total_steps": 28610, "tokens": 57608765440, "source": "finemath", "phase": 5, "loss": 2.1544594168663025, "ce_loss": 2.1426647901535034, "logit_z_loss": 35.92954611778259, "router_aux_loss": 1.1465794295072556, "router_z_loss": 0.3288428373634815, "routing_entropy_normalized": 0.7780944910776741, "learning_rate": 0.0004930431447630445, "grad_norm": 0.5189215540885925, "tokens_per_second": 4303604.344132448, "eta_seconds": 555.5234842300415, "data_wait_seconds": 2.1666288375854492e-05}
+{"step": 27480, "total_steps": 28610, "tokens": 57629736960, "source": "cosmopedia_v2", "phase": 5, "loss": 2.8108617663383484, "ce_loss": 2.7987828254699707, "logit_z_loss": 27.06228256225586, "router_aux_loss": 1.1470867544412613, "router_z_loss": 0.6081330478191376, "routing_entropy_normalized": 0.7456178443489713, "learning_rate": 0.0004849286417102539, "grad_norm": 0.6440252661705017, "tokens_per_second": 3676986.514099815, "eta_seconds": 644.4901962280273, "data_wait_seconds": 1.817941665649414e-05}
+{"step": 27490, "total_steps": 28610, "tokens": 57650708480, "source": "fineweb_edu", "phase": 5, "loss": 3.445550709962845, "ce_loss": 3.4333740770816803, "logit_z_loss": 24.65852451324463, "router_aux_loss": 1.1776469945907593, "router_z_loss": 0.40017206221818924, "routing_entropy_normalized": 0.6845603437335309, "learning_rate": 0.00047686857385382674, "grad_norm": 1.757568597793579, "tokens_per_second": 3689267.872285071, "eta_seconds": 636.6602592468262, "data_wait_seconds": 1.862645149230957e-05}
+{"step": 27500, "total_steps": 28610, "tokens": 57671680000, "source": "cosmopedia_v2", "phase": 5, "loss": 2.778052270412445, "ce_loss": 2.7656134963035583, "logit_z_loss": 27.1903395652771, "router_aux_loss": 1.1812237650156021, "router_z_loss": 0.6265436410903931, "routing_entropy_normalized": 0.700057633968754, "learning_rate": 0.000468863373430741, "grad_norm": 0.6669081449508667, "tokens_per_second": 4192282.1750717796, "eta_seconds": 555.2676615715027, "data_wait_seconds": 2.205371856689453e-05}
+{"step": 27510, "total_steps": 28610, "tokens": 57692651520, "source": "npset2", "phase": 5, "loss": 1.5720415860414505, "ce_loss": 1.5598599314689636, "logit_z_loss": 38.37356901168823, "router_aux_loss": 1.1775330305099487, "router_z_loss": 0.4063064828515053, "routing_entropy_normalized": 0.6838369700740539, "learning_rate": 0.0004609134697356009, "grad_norm": 0.5728282332420349, "tokens_per_second": 4627358.5007601185, "eta_seconds": 498.5278749465942, "data_wait_seconds": 2.0891427993774414e-05}
+{"step": 27520, "total_steps": 28610, "tokens": 57713623040, "source": "fineweb_edu", "phase": 5, "loss": 3.38229900598526, "ce_loss": 3.370233654975891, "logit_z_loss": 24.675041913986206, "router_aux_loss": 1.1649102419614792, "router_z_loss": 0.41629115864634514, "routing_entropy_normalized": 0.724451127390397, "learning_rate": 0.0004530192890976151, "grad_norm": 1.1351481676101685, "tokens_per_second": 3849666.519909384, "eta_seconds": 593.7905707359314, "data_wait_seconds": 2.4646520614624023e-05}
+{"step": 27530, "total_steps": 28610, "tokens": 57734594560, "source": "dclm", "phase": 5, "loss": 3.557970643043518, "ce_loss": 3.5460150241851807, "logit_z_loss": 24.621973991394043, "router_aux_loss": 1.1624541282653809, "router_z_loss": 0.3310382701456547, "routing_entropy_normalized": 0.7178517426099329, "learning_rate": 0.00044518125485773334, "grad_norm": 0.7012308239936829, "tokens_per_second": 3791067.2617297834, "eta_seconds": 597.4370813369751, "data_wait_seconds": 2.199411392211914e-05}
+{"step": 27540, "total_steps": 28610, "tokens": 57755566080, "source": "finemath", "phase": 5, "loss": 2.0547412633895874, "ce_loss": 2.042922481894493, "logit_z_loss": 36.06462812423706, "router_aux_loss": 1.1562795490026474, "router_z_loss": 0.25598170794546604, "routing_entropy_normalized": 0.7469674111532102, "learning_rate": 0.00043739978734594494, "grad_norm": 0.4908483326435089, "tokens_per_second": 4073321.7115883483, "eta_seconds": 550.8901085853577, "data_wait_seconds": 1.9609928131103516e-05}
+{"step": 27550, "total_steps": 28610, "tokens": 57776537600, "source": "dclm", "phase": 5, "loss": 3.564127117395401, "ce_loss": 3.5523250102996826, "logit_z_loss": 24.759324550628662, "router_aux_loss": 1.1525534242391586, "router_z_loss": 0.27655231580138206, "routing_entropy_normalized": 0.7257696931146084, "learning_rate": 0.00042967530385873697, "grad_norm": 0.47635790705680847, "tokens_per_second": 3970219.794752279, "eta_seconds": 559.9138674736023, "data_wait_seconds": 2.276897430419922e-05}
+{"step": 27560, "total_steps": 28610, "tokens": 57797509120, "source": "cosmopedia_v2", "phase": 5, "loss": 2.7337030470371246, "ce_loss": 2.721694737672806, "logit_z_loss": 27.768359899520874, "router_aux_loss": 1.142986610531807, "router_z_loss": 0.5784645825624466, "routing_entropy_normalized": 0.691453704430512, "learning_rate": 0.00042200821863671667, "grad_norm": 1.1140912771224976, "tokens_per_second": 3219577.010841031, "eta_seconds": 683.9437580108643, "data_wait_seconds": 2.187490463256836e-05}
+{"step": 27570, "total_steps": 28610, "tokens": 57818480640, "source": "finemath", "phase": 5, "loss": 2.1779327243566513, "ce_loss": 2.1662165820598602, "logit_z_loss": 36.34190368652344, "router_aux_loss": 1.1488913595676422, "router_z_loss": 0.22726093418896198, "routing_entropy_normalized": 0.7303074859835604, "learning_rate": 0.0004143989428423947, "grad_norm": 0.3000516891479492, "tokens_per_second": 4001469.1105673467, "eta_seconds": 545.0593318939209, "data_wait_seconds": 2.3305416107177734e-05}
+{"step": 27580, "total_steps": 28610, "tokens": 57839452160, "source": "cosmopedia_v2", "phase": 5, "loss": 2.767499625682831, "ce_loss": 2.755429983139038, "logit_z_loss": 27.709960460662842, "router_aux_loss": 1.14735846221447, "router_z_loss": 0.5960491895675659, "routing_entropy_normalized": 0.7326848285492075, "learning_rate": 0.0004068478845381405, "grad_norm": 0.584567666053772, "tokens_per_second": 4079096.452688106, "eta_seconds": 529.5453503131866, "data_wait_seconds": 1.9341707229614258e-05}
+{"step": 27590, "total_steps": 28610, "tokens": 57860423680, "source": "fineweb_edu", "phase": 5, "loss": 3.372015208005905, "ce_loss": 3.360063314437866, "logit_z_loss": 24.364545345306396, "router_aux_loss": 1.1558517664670944, "router_z_loss": 0.3934008814394474, "routing_entropy_normalized": 0.7002400413831071, "learning_rate": 0.0003993554486642944, "grad_norm": 1.0940029621124268, "tokens_per_second": 3615408.094000932, "eta_seconds": 591.6607432365417, "data_wait_seconds": 1.767277717590332e-05}
+{"step": 27600, "total_steps": 28610, "tokens": 57881395200, "source": "cosmopedia_v2", "phase": 5, "loss": 2.734100729227066, "ce_loss": 2.7219743728637695, "logit_z_loss": 28.118155002593994, "router_aux_loss": 1.1536212265491486, "router_z_loss": 0.590161457657814, "routing_entropy_normalized": 0.7194508000111869, "learning_rate": 0.0003919220370174532, "grad_norm": 1.1817651987075806, "tokens_per_second": 4327653.33472667, "eta_seconds": 489.43927717208857, "data_wait_seconds": 1.77919864654541e-05}
+{"step": 27610, "total_steps": 28610, "tokens": 57902366720, "source": "npset2", "phase": 5, "loss": 1.2614512294530869, "ce_loss": 1.2496893517673016, "logit_z_loss": 40.086236000061035, "router_aux_loss": 1.143567681312561, "router_z_loss": 0.3262030649930239, "routing_entropy_normalized": 0.7234530513302632, "learning_rate": 0.0003845480482289242, "grad_norm": 0.7926315069198608, "tokens_per_second": 4035048.133682905, "eta_seconds": 519.7340726852417, "data_wait_seconds": 1.9729137420654297e-05}
+{"step": 27620, "total_steps": 28610, "tokens": 57923338240, "source": "fineweb_edu", "phase": 5, "loss": 3.4163236916065216, "ce_loss": 3.4046172499656677, "logit_z_loss": 24.13103175163269, "router_aux_loss": 1.1362032890319824, "router_z_loss": 0.3443964868783951, "routing_entropy_normalized": 0.7433476527433595, "learning_rate": 0.0003772338777433482, "grad_norm": 0.7691269516944885, "tokens_per_second": 3953571.565207194, "eta_seconds": 525.1404826641083, "data_wait_seconds": 1.901388168334961e-05}
+{"step": 27630, "total_steps": 28610, "tokens": 57944309760, "source": "dclm", "phase": 5, "loss": 3.551723897457123, "ce_loss": 3.540062725543976, "logit_z_loss": 24.651623249053955, "router_aux_loss": 1.136534333229065, "router_z_loss": 0.2958680745214224, "routing_entropy_normalized": 0.7622628424578234, "learning_rate": 0.00036997991779749115, "grad_norm": 0.4423021376132965, "tokens_per_second": 4340584.122259875, "eta_seconds": 473.4867248535156, "data_wait_seconds": 2.199411392211914e-05}
+{"step": 27640, "total_steps": 28610, "tokens": 57965281280, "source": "finemath", "phase": 5, "loss": 2.150364711880684, "ce_loss": 2.1388886272907257, "logit_z_loss": 36.80145883560181, "router_aux_loss": 1.1270176768302917, "router_z_loss": 0.20588162541389465, "routing_entropy_normalized": 0.762633322775464, "learning_rate": 0.0003627865573992087, "grad_norm": 0.4371199905872345, "tokens_per_second": 3830269.065808735, "eta_seconds": 531.095180273056, "data_wait_seconds": 1.6570091247558594e-05}
+{"step": 27650, "total_steps": 28610, "tokens": 57986252800, "source": "dclm", "phase": 5, "loss": 3.4955612421035767, "ce_loss": 3.4838794469833374, "logit_z_loss": 25.015810251235962, "router_aux_loss": 1.1396625339984894, "router_z_loss": 0.2851655278354883, "routing_entropy_normalized": 0.7506955811748501, "learning_rate": 0.0003556541823065897, "grad_norm": 0.4225808084011078, "tokens_per_second": 3739549.1196477683, "eta_seconds": 538.3712997436523, "data_wait_seconds": 1.633167266845703e-05}
+{"step": 27660, "total_steps": 28610, "tokens": 58007224320, "source": "cosmopedia_v2", "phase": 5, "loss": 2.7223654091358185, "ce_loss": 2.710407465696335, "logit_z_loss": 27.782416582107544, "router_aux_loss": 1.1393316090106964, "router_z_loss": 0.5645734593272209, "routing_entropy_normalized": 0.7182731258910863, "learning_rate": 0.0003485831750072649, "grad_norm": 0.3914169669151306, "tokens_per_second": 4182727.440512197, "eta_seconds": 476.31466031074524, "data_wait_seconds": 2.3573637008666992e-05}
+{"step": 27670, "total_steps": 28610, "tokens": 58028195840, "source": "finemath", "phase": 5, "loss": 2.1604300886392593, "ce_loss": 2.1488812416791916, "logit_z_loss": 37.02727127075195, "router_aux_loss": 1.1365265995264053, "router_z_loss": 0.18357402086257935, "routing_entropy_normalized": 0.7678460878578244, "learning_rate": 0.0003415739146978952, "grad_norm": 0.5129936337471008, "tokens_per_second": 3729616.446420361, "eta_seconds": 528.5591449737549, "data_wait_seconds": 1.8328428268432617e-05}
+{"step": 27680, "total_steps": 28610, "tokens": 58049167360, "source": "cosmopedia_v2", "phase": 5, "loss": 2.717371940612793, "ce_loss": 2.7055833637714386, "logit_z_loss": 27.751835107803345, "router_aux_loss": 1.1230333000421524, "router_z_loss": 0.5583273842930794, "routing_entropy_normalized": 0.7408505607367132, "learning_rate": 0.0003346267772638392, "grad_norm": 0.29500848054885864, "tokens_per_second": 4101095.80167952, "eta_seconds": 475.5683491230011, "data_wait_seconds": 2.372264862060547e-05}
+{"step": 27690, "total_steps": 28610, "tokens": 58070138880, "source": "fineweb_edu", "phase": 5, "loss": 3.338480919599533, "ce_loss": 3.326885849237442, "logit_z_loss": 24.668848514556885, "router_aux_loss": 1.1261379271745682, "router_z_loss": 0.333767581731081, "routing_entropy_normalized": 0.7506478305453763, "learning_rate": 0.00032774213525899493, "grad_norm": 1.0992536544799805, "tokens_per_second": 3938926.2519126274, "eta_seconds": 489.82380390167236, "data_wait_seconds": 2.3156404495239258e-05}
+{"step": 27700, "total_steps": 28610, "tokens": 58091110400, "source": "cosmopedia_v2", "phase": 5, "loss": 2.714723616838455, "ce_loss": 2.7029493153095245, "logit_z_loss": 27.913459539413452, "router_aux_loss": 1.1235522478818893, "router_z_loss": 0.5387637987732887, "routing_entropy_normalized": 0.7353492922296287, "learning_rate": 0.00032092035788581905, "grad_norm": 0.39336878061294556, "tokens_per_second": 4516121.640423276, "eta_seconds": 422.5768196582794, "data_wait_seconds": 2.1845102310180664e-05}
+{"step": 27710, "total_steps": 28610, "tokens": 58112081920, "source": "npset2", "phase": 5, "loss": 1.51713465154171, "ce_loss": 1.5055192708969116, "logit_z_loss": 39.36193656921387, "router_aux_loss": 1.126556396484375, "router_z_loss": 0.34981443360447884, "routing_entropy_normalized": 0.71747869100081, "learning_rate": 0.00031416181097552846, "grad_norm": 0.7251936197280884, "tokens_per_second": 3673098.7623040825, "eta_seconds": 513.8540840148926, "data_wait_seconds": 1.8090009689331055e-05}
+{"step": 27720, "total_steps": 28610, "tokens": 58133053440, "source": "fineweb_edu", "phase": 5, "loss": 3.383846491575241, "ce_loss": 3.371325343847275, "logit_z_loss": 22.65258002281189, "router_aux_loss": 1.2194788306951523, "router_z_loss": 0.3263409584760666, "routing_entropy_normalized": 0.6414140431461011, "learning_rate": 0.0003074668569684829, "grad_norm": 3.9371461868286133, "tokens_per_second": 3904019.5762621253, "eta_seconds": 478.08809447288513, "data_wait_seconds": 2.2917985916137695e-05}
+{"step": 27730, "total_steps": 28610, "tokens": 58154024960, "source": "dclm", "phase": 5, "loss": 3.4980911016464233, "ce_loss": 3.4848386645317078, "logit_z_loss": 24.763427257537842, "router_aux_loss": 1.3017573356628418, "router_z_loss": 0.23484240472316742, "routing_entropy_normalized": 0.6167399530235155, "learning_rate": 0.0003008358548947486, "grad_norm": 1.8664525747299194, "tokens_per_second": 4210482.766861056, "eta_seconds": 438.3093013763427, "data_wait_seconds": 2.053380012512207e-05}
+{"step": 27740, "total_steps": 28610, "tokens": 58174996480, "source": "finemath", "phase": 5, "loss": 2.1135820001363754, "ce_loss": 2.1014843434095383, "logit_z_loss": 37.0741286277771, "router_aux_loss": 1.1896660178899765, "router_z_loss": 0.2010026052594185, "routing_entropy_normalized": 0.6748067982159978, "learning_rate": 0.00029426916035484165, "grad_norm": 0.6280632019042969, "tokens_per_second": 3610586.0072669047, "eta_seconds": 505.32579374313354, "data_wait_seconds": 1.665949821472168e-05}
+{"step": 27750, "total_steps": 28610, "tokens": 58195968000, "source": "dclm", "phase": 5, "loss": 3.49196395277977, "ce_loss": 3.4802332520484924, "logit_z_loss": 24.838901042938232, "router_aux_loss": 1.1450781673192978, "router_z_loss": 0.27989829145371914, "routing_entropy_normalized": 0.7085990260586701, "learning_rate": 0.0002877671255006612, "grad_norm": 0.48371610045433044, "tokens_per_second": 4035806.2627049014, "eta_seconds": 446.88733863830566, "data_wait_seconds": 2.321600914001465e-05}
+{"step": 27760, "total_steps": 28610, "tokens": 58216939520, "source": "cosmopedia_v2", "phase": 5, "loss": 2.701373368501663, "ce_loss": 2.6894377768039703, "logit_z_loss": 27.893439531326294, "router_aux_loss": 1.1392162442207336, "router_z_loss": 0.5434527173638344, "routing_entropy_normalized": 0.7031680987134749, "learning_rate": 0.00028133009901660497, "grad_norm": 0.6565626263618469, "tokens_per_second": 3870674.6393131325, "eta_seconds": 460.53449749946594, "data_wait_seconds": 2.2798776626586914e-05}
+{"step": 27770, "total_steps": 28610, "tokens": 58237911040, "source": "finemath", "phase": 5, "loss": 2.2587705552577972, "ce_loss": 2.2472039461135864, "logit_z_loss": 35.48642158508301, "router_aux_loss": 1.1383630335330963, "router_z_loss": 0.18299024552106857, "routing_entropy_normalized": 0.7165015638687376, "learning_rate": 0.0002749584261008676, "grad_norm": 0.24051184952259064, "tokens_per_second": 3791855.4695031308, "eta_seconds": 464.5766944885254, "data_wait_seconds": 2.244114875793457e-05}
+{"step": 27780, "total_steps": 28610, "tokens": 58258882560, "source": "cosmopedia_v2", "phase": 5, "loss": 2.705506354570389, "ce_loss": 2.6936777532100677, "logit_z_loss": 27.664756298065186, "router_aux_loss": 1.1251948922872543, "router_z_loss": 0.5766317918896675, "routing_entropy_normalized": 0.75516974607052, "learning_rate": 0.00026865244844692996, "grad_norm": 0.4140530228614807, "tokens_per_second": 3642861.6248250026, "eta_seconds": 477.82110309600836, "data_wait_seconds": 1.9788742065429688e-05}
+{"step": 27790, "total_steps": 28610, "tokens": 58279854080, "source": "fineweb_edu", "phase": 5, "loss": 3.3298254907131195, "ce_loss": 3.3182373344898224, "logit_z_loss": 24.252888202667236, "router_aux_loss": 1.1213261038064957, "router_z_loss": 0.3748525194823742, "routing_entropy_normalized": 0.7373438245623659, "learning_rate": 0.00026241250422523585, "grad_norm": 0.4339379668235779, "tokens_per_second": 4240435.0527585335, "eta_seconds": 405.53967189788824, "data_wait_seconds": 2.0444393157958984e-05}
+{"step": 27800, "total_steps": 28610, "tokens": 58300825600, "source": "cosmopedia_v2", "phase": 5, "loss": 2.7063998579978943, "ce_loss": 2.6945784986019135, "logit_z_loss": 27.460379362106323, "router_aux_loss": 1.1271987110376358, "router_z_loss": 0.5493593290448189, "routing_entropy_normalized": 0.7464171700833936, "learning_rate": 0.00025623892806505774, "grad_norm": 0.9743544459342957, "tokens_per_second": 3802096.249596766, "eta_seconds": 446.77804255485535, "data_wait_seconds": 2.5600194931030273e-05}
+{"step": 27810, "total_steps": 28610, "tokens": 58321797120, "source": "npset2", "phase": 5, "loss": 1.6617633402347565, "ce_loss": 1.6501868814229965, "logit_z_loss": 38.33955669403076, "router_aux_loss": 1.1166395992040634, "router_z_loss": 0.4100771136581898, "routing_entropy_normalized": 0.7689260357192811, "learning_rate": 0.0002501320510365484, "grad_norm": 0.5131340026855469, "tokens_per_second": 4103158.0792142767, "eta_seconds": 408.885440826416, "data_wait_seconds": 2.288818359375e-05}
+{"step": 27820, "total_steps": 28610, "tokens": 58342768640, "source": "fineweb_edu", "phase": 5, "loss": 3.369535267353058, "ce_loss": 3.357931226491928, "logit_z_loss": 24.21927046775818, "router_aux_loss": 1.1260649859905243, "router_z_loss": 0.3433970510959625, "routing_entropy_normalized": 0.7448027472894088, "learning_rate": 0.00024409220063298953, "grad_norm": 0.25202175974845886, "tokens_per_second": 3793870.377210933, "eta_seconds": 436.6912717819214, "data_wait_seconds": 2.1785497665405273e-05}
+{"step": 27830, "total_steps": 28610, "tokens": 58363740160, "source": "dclm", "phase": 5, "loss": 3.4526363015174866, "ce_loss": 3.441180497407913, "logit_z_loss": 24.910789728164673, "router_aux_loss": 1.114931970834732, "router_z_loss": 0.306452177464962, "routing_entropy_normalized": 0.783133231510063, "learning_rate": 0.00023811970075322803, "grad_norm": 0.3993011713027954, "tokens_per_second": 3991176.0469389283, "eta_seconds": 409.8487615585327, "data_wait_seconds": 2.047419548034668e-05}
+{"step": 27840, "total_steps": 28610, "tokens": 58384711680, "source": "finemath", "phase": 5, "loss": 2.1693107187747955, "ce_loss": 2.1578984558582306, "logit_z_loss": 36.06208324432373, "router_aux_loss": 1.120962604880333, "router_z_loss": 0.20255792699754238, "routing_entropy_normalized": 0.7675690151741489, "learning_rate": 0.0002322148716843081, "grad_norm": 0.5056644678115845, "tokens_per_second": 3507412.709255468, "eta_seconds": 460.3983545303345, "data_wait_seconds": 1.7464160919189453e-05}
+{"step": 27850, "total_steps": 28610, "tokens": 58405683200, "source": "dclm", "phase": 5, "loss": 3.442443937063217, "ce_loss": 3.430951237678528, "logit_z_loss": 24.61332082748413, "router_aux_loss": 1.117176428437233, "router_z_loss": 0.3208998218178749, "routing_entropy_normalized": 0.781347691420392, "learning_rate": 0.00022637803008429102, "grad_norm": 0.41088104248046875, "tokens_per_second": 3940437.760358782, "eta_seconds": 404.4818410873413, "data_wait_seconds": 2.0116567611694336e-05}
+{"step": 27860, "total_steps": 28610, "tokens": 58426654720, "source": "cosmopedia_v2", "phase": 5, "loss": 2.6912881433963776, "ce_loss": 2.679574340581894, "logit_z_loss": 27.730440616607666, "router_aux_loss": 1.117054060101509, "router_z_loss": 0.5432156659662724, "routing_entropy_normalized": 0.7575764777570495, "learning_rate": 0.00022060948896527872, "grad_norm": 0.777539849281311, "tokens_per_second": 3844382.9060194246, "eta_seconds": 409.13302302360535, "data_wait_seconds": 2.0265579223632812e-05}
+{"step": 27870, "total_steps": 28610, "tokens": 58447626240, "source": "finemath", "phase": 5, "loss": 2.1718658953905106, "ce_loss": 2.160458266735077, "logit_z_loss": 35.50574445724487, "router_aux_loss": 1.118214949965477, "router_z_loss": 0.2255100030452013, "routing_entropy_normalized": 0.7803306428710973, "learning_rate": 0.00021490955767662447, "grad_norm": 0.5993155241012573, "tokens_per_second": 3974727.4993909867, "eta_seconds": 390.439968585968, "data_wait_seconds": 1.6361474990844727e-05}
+{"step": 27880, "total_steps": 28610, "tokens": 58468597760, "source": "cosmopedia_v2", "phase": 5, "loss": 2.7010094821453094, "ce_loss": 2.6892242431640625, "logit_z_loss": 27.78027868270874, "router_aux_loss": 1.122684195637703, "router_z_loss": 0.5583646595478058, "routing_entropy_normalized": 0.7400205049227344, "learning_rate": 0.00020927854188834507, "grad_norm": 1.039543628692627, "tokens_per_second": 3853110.0227408186, "eta_seconds": 397.32085275650024, "data_wait_seconds": 1.811981201171875e-05}
+{"step": 27890, "total_steps": 28610, "tokens": 58489569280, "source": "fineweb_edu", "phase": 5, "loss": 3.3493749499320984, "ce_loss": 3.3379043638706207, "logit_z_loss": 24.344539403915405, "router_aux_loss": 1.1134139150381088, "router_z_loss": 0.3364297114312649, "routing_entropy_normalized": 0.7723528111205986, "learning_rate": 0.0002037167435747279, "grad_norm": 0.2564642131328583, "tokens_per_second": 4069246.321748795, "eta_seconds": 371.0636615753174, "data_wait_seconds": 2.378225326538086e-05}
+{"step": 27900, "total_steps": 28610, "tokens": 58510540800, "source": "cosmopedia_v2", "phase": 5, "loss": 2.695162355899811, "ce_loss": 2.6831369400024414, "logit_z_loss": 27.76621913909912, "router_aux_loss": 1.14834263920784, "router_z_loss": 0.541979543864727, "routing_entropy_normalized": 0.6718089782434407, "learning_rate": 0.00019822446099813685, "grad_norm": 0.5913159847259521, "tokens_per_second": 4416124.579488824, "eta_seconds": 337.1684591770172, "data_wait_seconds": 1.9729137420654297e-05}
+{"step": 27910, "total_steps": 28610, "tokens": 58531512320, "source": "npset2", "phase": 5, "loss": 1.6481776684522629, "ce_loss": 1.6365756690502167, "logit_z_loss": 36.79128694534302, "router_aux_loss": 1.1228550970554352, "router_z_loss": 0.37343648076057434, "routing_entropy_normalized": 0.7884279792222472, "learning_rate": 0.0001928019886930197, "grad_norm": 0.670244038105011, "tokens_per_second": 3944093.5681051575, "eta_seconds": 372.20374584198, "data_wait_seconds": 1.671910285949707e-05}
+{"step": 27920, "total_steps": 28610, "tokens": 58552483840, "source": "fineweb_edu", "phase": 5, "loss": 3.3459804356098175, "ce_loss": 3.3339628875255585, "logit_z_loss": 24.279399394989014, "router_aux_loss": 1.1681936830282211, "router_z_loss": 0.3355780802667141, "routing_entropy_normalized": 0.6368045870925783, "learning_rate": 0.00018744961745010842, "grad_norm": 0.5130188465118408, "tokens_per_second": 4361349.895019809, "eta_seconds": 331.78601002693176, "data_wait_seconds": 2.345442771911621e-05}
+{"step": 27930, "total_steps": 28610, "tokens": 58573455360, "source": "dclm", "phase": 5, "loss": 3.5838514864444733, "ce_loss": 3.57213494181633, "logit_z_loss": 24.150202989578247, "router_aux_loss": 1.1447164416313171, "router_z_loss": 0.26937744580209255, "routing_entropy_normalized": 0.7283975053459159, "learning_rate": 0.00018216763430082983, "grad_norm": 1.0059657096862793, "tokens_per_second": 4125485.966980515, "eta_seconds": 345.6716060638428, "data_wait_seconds": 2.4378299713134766e-05}
+{"step": 27940, "total_steps": 28610, "tokens": 58594426880, "source": "finemath", "phase": 5, "loss": 2.2731394469738007, "ce_loss": 2.261336535215378, "logit_z_loss": 34.64069843292236, "router_aux_loss": 1.161469280719757, "router_z_loss": 0.18824726343154907, "routing_entropy_normalized": 0.742558525597635, "learning_rate": 0.00017695632250191003, "grad_norm": 0.8087918758392334, "tokens_per_second": 4058464.8166556233, "eta_seconds": 346.2126452922821, "data_wait_seconds": 1.7017126083374023e-05}
+{"step": 27950, "total_steps": 28610, "tokens": 58615398400, "source": "dclm", "phase": 5, "loss": 3.752820521593094, "ce_loss": 3.7409128546714783, "logit_z_loss": 21.282929182052612, "router_aux_loss": 1.1658814400434494, "router_z_loss": 0.24882402829825878, "routing_entropy_normalized": 0.7707282105474644, "learning_rate": 0.0001718159615201853, "grad_norm": 0.4362967014312744, "tokens_per_second": 3918430.081378731, "eta_seconds": 353.2333846092224, "data_wait_seconds": 2.0325183868408203e-05}
+{"step": 27960, "total_steps": 28610, "tokens": 58636369920, "source": "cosmopedia_v2", "phase": 5, "loss": 3.0612345337867737, "ce_loss": 3.0490361154079437, "logit_z_loss": 24.539011001586914, "router_aux_loss": 1.171160340309143, "router_z_loss": 0.4867748022079468, "routing_entropy_normalized": 0.7344704468204011, "learning_rate": 0.00016674682701761496, "grad_norm": 0.7010442018508911, "tokens_per_second": 3943809.3909322484, "eta_seconds": 345.6426680088043, "data_wait_seconds": 1.9431114196777344e-05}
+{"step": 27970, "total_steps": 28610, "tokens": 58657341440, "source": "finemath", "phase": 5, "loss": 2.265582114458084, "ce_loss": 2.2536762058734894, "logit_z_loss": 34.183842182159424, "router_aux_loss": 1.1686714142560959, "router_z_loss": 0.21920132264494896, "routing_entropy_normalized": 0.7689684164417554, "learning_rate": 0.0001617491908364983, "grad_norm": 0.3635607957839966, "tokens_per_second": 4006828.765778145, "eta_seconds": 334.9724578857422, "data_wait_seconds": 2.0772218704223633e-05}
+{"step": 27980, "total_steps": 28610, "tokens": 58678312960, "source": "cosmopedia_v2", "phase": 5, "loss": 2.9660346508026123, "ce_loss": 2.953992784023285, "logit_z_loss": 25.201317310333252, "router_aux_loss": 1.1566181927919388, "router_z_loss": 0.47574447840452194, "routing_entropy_normalized": 0.7758726536495058, "learning_rate": 0.0001568233209848966, "grad_norm": 0.3331252634525299, "tokens_per_second": 4150104.5334071624, "eta_seconds": 318.3548147678375, "data_wait_seconds": 2.047419548034668e-05}
+{"step": 27990, "total_steps": 28610, "tokens": 58699284480, "source": "fineweb_edu", "phase": 5, "loss": 3.5624370872974396, "ce_loss": 3.550575315952301, "logit_z_loss": 21.68261981010437, "router_aux_loss": 1.1555361300706863, "router_z_loss": 0.3063991069793701, "routing_entropy_normalized": 0.8089577601627074, "learning_rate": 0.00015196948162226115, "grad_norm": 0.3019684851169586, "tokens_per_second": 3850933.5929408353, "eta_seconds": 337.6413040161133, "data_wait_seconds": 2.0682811737060547e-05}
+{"step": 28000, "total_steps": 28610, "tokens": 58720256000, "source": "cosmopedia_v2", "phase": 5, "loss": 2.925056040287018, "ce_loss": 2.9130779206752777, "logit_z_loss": 25.42520260810852, "router_aux_loss": 1.1506331413984299, "router_z_loss": 0.4718184284865856, "routing_entropy_normalized": 0.8409485850295259, "learning_rate": 0.00014718793304526584, "grad_norm": 0.3107222616672516, "tokens_per_second": 3554947.515250123, "eta_seconds": 359.85417914390564, "data_wait_seconds": 2.1785497665405273e-05}
+{"step": 28010, "total_steps": 28610, "tokens": 58741227520, "source": "npset2", "phase": 5, "loss": 1.5751665085554123, "ce_loss": 1.5632983893156052, "logit_z_loss": 37.706584453582764, "router_aux_loss": 1.1509435176849365, "router_z_loss": 0.3586685322225094, "routing_entropy_normalized": 0.772169756708347, "learning_rate": 0.00014247893167384978, "grad_norm": 0.316896915435791, "tokens_per_second": 4309754.721011988, "eta_seconds": 291.963529586792, "data_wait_seconds": 1.9550323486328125e-05}
+{"step": 28020, "total_steps": 28610, "tokens": 58762199040, "source": "fineweb_edu", "phase": 5, "loss": 3.5036895275115967, "ce_loss": 3.4918562173843384, "logit_z_loss": 22.20421814918518, "router_aux_loss": 1.1515648663043976, "router_z_loss": 0.31770576909184456, "routing_entropy_normalized": 0.8071559160352636, "learning_rate": 0.00013784273003746567, "grad_norm": 0.44447430968284607, "tokens_per_second": 4509792.633359326, "eta_seconds": 274.36287665367126, "data_wait_seconds": 2.327561378479004e-05}
+{"step": 28030, "total_steps": 28610, "tokens": 58783170560, "source": "dclm", "phase": 5, "loss": 3.630942702293396, "ce_loss": 3.6191861033439636, "logit_z_loss": 22.50275158882141, "router_aux_loss": 1.1531541794538498, "router_z_loss": 0.2251303791999817, "routing_entropy_normalized": 0.7635803105731761, "learning_rate": 0.0001332795767615369, "grad_norm": 0.28792816400527954, "tokens_per_second": 3671684.0484204683, "eta_seconds": 331.2780032157898, "data_wait_seconds": 2.1904706954956055e-05}
+{"step": 28040, "total_steps": 28610, "tokens": 58804142080, "source": "finemath", "phase": 5, "loss": 2.242888778448105, "ce_loss": 2.2311703264713287, "logit_z_loss": 34.46228313446045, "router_aux_loss": 1.1514662355184555, "router_z_loss": 0.20383991114795208, "routing_entropy_normalized": 0.8179247004865856, "learning_rate": 0.00012878971655412515, "grad_norm": 0.3706262409687042, "tokens_per_second": 4233341.17597278, "eta_seconds": 282.3719115257263, "data_wait_seconds": 1.6510486602783203e-05}
+{"step": 28050, "total_steps": 28610, "tokens": 58825113600, "source": "dclm", "phase": 5, "loss": 3.598474830389023, "ce_loss": 3.5867534279823303, "logit_z_loss": 22.78657031059265, "router_aux_loss": 1.15047787129879, "router_z_loss": 0.21663139574229717, "routing_entropy_normalized": 0.8420933047150662, "learning_rate": 0.00012437339019280742, "grad_norm": 0.24644261598587036, "tokens_per_second": 4272765.397790203, "eta_seconds": 274.85832023620605, "data_wait_seconds": 1.7136335372924805e-05}
+{"step": 28060, "total_steps": 28610, "tokens": 58846085120, "source": "cosmopedia_v2", "phase": 5, "loss": 2.857569843530655, "ce_loss": 2.8455983996391296, "logit_z_loss": 25.779847383499146, "router_aux_loss": 1.1471699625253677, "router_z_loss": 0.49973373487591743, "routing_entropy_normalized": 0.8107803853481972, "learning_rate": 0.00012003083451176366, "grad_norm": 0.5798690319061279, "tokens_per_second": 3934000.1185232466, "eta_seconds": 293.19612741470337, "data_wait_seconds": 1.96993350982666e-05}
+{"step": 28070, "total_steps": 28610, "tokens": 58867056640, "source": "finemath", "phase": 5, "loss": 2.2018981873989105, "ce_loss": 2.1902053356170654, "logit_z_loss": 34.48916673660278, "router_aux_loss": 1.1503749340772629, "router_z_loss": 0.18907971866428852, "routing_entropy_normalized": 0.8204539886825669, "learning_rate": 0.00011576228238907605, "grad_norm": 0.2141193151473999, "tokens_per_second": 3933840.7180802342, "eta_seconds": 287.876953125, "data_wait_seconds": 1.8924474716186523e-05}
+{"step": 28080, "total_steps": 28610, "tokens": 58888028160, "source": "cosmopedia_v2", "phase": 5, "loss": 2.856248199939728, "ce_loss": 2.844309836626053, "logit_z_loss": 25.93678569793701, "router_aux_loss": 1.144879698753357, "router_z_loss": 0.4895361140370369, "routing_entropy_normalized": 0.8318042670321977, "learning_rate": 0.00011156796273424058, "grad_norm": 0.5318847298622131, "tokens_per_second": 4005079.522803191, "eta_seconds": 277.5202224254608, "data_wait_seconds": 1.5616416931152344e-05}
+{"step": 28090, "total_steps": 28610, "tokens": 58908999680, "source": "fineweb_edu", "phase": 5, "loss": 3.4692002832889557, "ce_loss": 3.457459509372711, "logit_z_loss": 22.52413058280945, "router_aux_loss": 1.1444062143564224, "router_z_loss": 0.29670859687030315, "routing_entropy_normalized": 0.826239041922377, "learning_rate": 0.00010744810047589115, "grad_norm": 0.32552576065063477, "tokens_per_second": 3668501.365980148, "eta_seconds": 297.26554012298584, "data_wait_seconds": 0.053449004888534546}
+{"step": 28100, "total_steps": 28610, "tokens": 58929971200, "source": "cosmopedia_v2", "phase": 5, "loss": 2.8385543525218964, "ce_loss": 2.826692223548889, "logit_z_loss": 26.14983296394348, "router_aux_loss": 1.1440565586090088, "router_z_loss": 0.4216091223061085, "routing_entropy_normalized": 0.8469309961037502, "learning_rate": 0.0001034029165497375, "grad_norm": 0.8730766177177429, "tokens_per_second": 4261380.212448583, "eta_seconds": 250.98617506027222, "data_wait_seconds": 1.9997358322143555e-05}
+{"step": 28110, "total_steps": 28610, "tokens": 58950942720, "source": "npset2", "phase": 5, "loss": 1.5759007185697556, "ce_loss": 1.5641705244779587, "logit_z_loss": 37.4142427444458, "router_aux_loss": 1.142189860343933, "router_z_loss": 0.30830928683280945, "routing_entropy_normalized": 0.8065408826439442, "learning_rate": 9.943262788671748e-05, "grad_norm": 0.3037864565849304, "tokens_per_second": 4231393.501198642, "eta_seconds": 247.8086709976196, "data_wait_seconds": 1.952052116394043e-05}
+{"step": 28120, "total_steps": 28610, "tokens": 58971914240, "source": "fineweb_edu", "phase": 5, "loss": 3.4753189086914062, "ce_loss": 3.463602304458618, "logit_z_loss": 22.915748834609985, "router_aux_loss": 1.1444860845804214, "router_z_loss": 0.2718108557164669, "routing_entropy_normalized": 0.8356689061221874, "learning_rate": 9.553744740136211e-05, "grad_norm": 0.4054260849952698, "tokens_per_second": 4047203.3560181684, "eta_seconds": 253.90482997894287, "data_wait_seconds": 2.142786979675293e-05}
+{"step": 28130, "total_steps": 28610, "tokens": 58992885760, "source": "dclm", "phase": 5, "loss": 3.5528814792633057, "ce_loss": 3.541232615709305, "logit_z_loss": 23.04280376434326, "router_aux_loss": 1.143473595380783, "router_z_loss": 0.21409964747726917, "routing_entropy_normalized": 0.8401052949936209, "learning_rate": 9.171758398038016e-05, "grad_norm": 0.2004852294921875, "tokens_per_second": 4335497.850071237, "eta_seconds": 232.1839370727539, "data_wait_seconds": 2.372264862060547e-05}
+{"step": 28140, "total_steps": 28610, "tokens": 59013857280, "source": "finemath", "phase": 5, "loss": 2.2527211606502533, "ce_loss": 2.2410978078842163, "logit_z_loss": 34.34602642059326, "router_aux_loss": 1.1417751461267471, "router_z_loss": 0.20564361847937107, "routing_entropy_normalized": 0.8369681773494698, "learning_rate": 8.797324247145411e-05, "grad_norm": 0.1915145367383957, "tokens_per_second": 4076619.1550633404, "eta_seconds": 241.78403782844543, "data_wait_seconds": 1.6421079635620117e-05}
+{"step": 28150, "total_steps": 28610, "tokens": 59034828800, "source": "dclm", "phase": 5, "loss": 3.565601944923401, "ce_loss": 3.5539133548736572, "logit_z_loss": 23.07322669029236, "router_aux_loss": 1.1441297382116318, "router_z_loss": 0.24734466150403023, "routing_entropy_normalized": 0.8211217550213764, "learning_rate": 8.430462367225562e-05, "grad_norm": 0.31337159872055054, "tokens_per_second": 3548874.05624914, "eta_seconds": 271.82985496520996, "data_wait_seconds": 1.8656253814697266e-05}
+{"step": 28160, "total_steps": 28610, "tokens": 59055800320, "source": "cosmopedia_v2", "phase": 5, "loss": 2.8249200880527496, "ce_loss": 2.8130362033843994, "logit_z_loss": 26.173656225204468, "router_aux_loss": 1.1436467468738556, "router_z_loss": 0.4474475719034672, "routing_entropy_normalized": 0.8230507695401179, "learning_rate": 8.07119243196771e-05, "grad_norm": 0.35555535554885864, "tokens_per_second": 3984821.6966118594, "eta_seconds": 236.82826280593872, "data_wait_seconds": 2.0742416381835938e-05}
+{"step": 28170, "total_steps": 28610, "tokens": 59076771840, "source": "finemath", "phase": 5, "loss": 2.2011674344539642, "ce_loss": 2.189516097307205, "logit_z_loss": 34.232972621917725, "router_aux_loss": 1.1433580070734024, "router_z_loss": 0.2177581014111638, "routing_entropy_normalized": 0.8152192616965523, "learning_rate": 7.719533707928179e-05, "grad_norm": 0.26549670100212097, "tokens_per_second": 3777935.108941766, "eta_seconds": 244.2463550567627, "data_wait_seconds": 0.0808371901512146}
+{"step": 28180, "total_steps": 28610, "tokens": 59097743360, "source": "cosmopedia_v2", "phase": 5, "loss": 2.825288772583008, "ce_loss": 2.813451737165451, "logit_z_loss": 26.08385133743286, "router_aux_loss": 1.1382818520069122, "router_z_loss": 0.4542282111942768, "routing_entropy_normalized": 0.8599172729154096, "learning_rate": 7.375505053497094e-05, "grad_norm": 0.2970309257507324, "tokens_per_second": 4345717.338486745, "eta_seconds": 207.5089771747589, "data_wait_seconds": 2.0235776901245117e-05}
+{"step": 28190, "total_steps": 28610, "tokens": 59118714880, "source": "fineweb_edu", "phase": 5, "loss": 3.46229550242424, "ce_loss": 3.450626254081726, "logit_z_loss": 22.68706464767456, "router_aux_loss": 1.1384910643100739, "router_z_loss": 0.28434498608112335, "routing_entropy_normalized": 0.8552725566802813, "learning_rate": 7.039124917887085e-05, "grad_norm": 0.21015027165412903, "tokens_per_second": 4285201.72301092, "eta_seconds": 205.54547882080075, "data_wait_seconds": 1.823902130126953e-05}
+{"step": 28200, "total_steps": 28610, "tokens": 59139686400, "source": "cosmopedia_v2", "phase": 5, "loss": 2.8146170675754547, "ce_loss": 2.8028166592121124, "logit_z_loss": 26.22274374961853, "router_aux_loss": 1.1355251520872116, "router_z_loss": 0.4451347477734089, "routing_entropy_normalized": 0.8575343142287454, "learning_rate": 6.710411340144007e-05, "grad_norm": 0.3364868760108948, "tokens_per_second": 4005561.196690192, "eta_seconds": 214.65963888168335, "data_wait_seconds": 2.1845102310180664e-05}
+{"step": 28210, "total_steps": 28610, "tokens": 59160657920, "source": "npset2", "phase": 5, "loss": 1.5706481337547302, "ce_loss": 1.5589661598205566, "logit_z_loss": 37.577370166778564, "router_aux_loss": 1.1358884572982788, "router_z_loss": 0.323131050914526, "routing_entropy_normalized": 0.8080924046325687, "learning_rate": 6.38938194817944e-05, "grad_norm": 0.30726292729377747, "tokens_per_second": 3628005.4402215914, "eta_seconds": 231.21817588806152, "data_wait_seconds": 1.895427703857422e-05}
+{"step": 28220, "total_steps": 28610, "tokens": 59181629440, "source": "fineweb_edu", "phase": 5, "loss": 3.3995717465877533, "ce_loss": 3.3878896236419678, "logit_z_loss": 23.089441299438477, "router_aux_loss": 1.1390522867441177, "router_z_loss": 0.2916587386280298, "routing_entropy_normalized": 0.8531743808163216, "learning_rate": 6.0760539578254115e-05, "grad_norm": 0.22132571041584015, "tokens_per_second": 4542284.952868354, "eta_seconds": 180.06120014190674, "data_wait_seconds": 1.940131187438965e-05}
+{"step": 28230, "total_steps": 28610, "tokens": 59202600960, "source": "dclm", "phase": 5, "loss": 3.5667548179626465, "ce_loss": 3.5551679134368896, "logit_z_loss": 23.128688097000122, "router_aux_loss": 1.1363331079483032, "router_z_loss": 0.22358847223222256, "routing_entropy_normalized": 0.8420471635179393, "learning_rate": 5.770444171911082e-05, "grad_norm": 0.15331889688968658, "tokens_per_second": 4196159.820767289, "eta_seconds": 189.91596937179565, "data_wait_seconds": 1.8537044525146484e-05}
+{"step": 28240, "total_steps": 28610, "tokens": 59223572480, "source": "finemath", "phase": 5, "loss": 2.2859044671058655, "ce_loss": 2.2743550539016724, "logit_z_loss": 34.651061058044434, "router_aux_loss": 1.1381640285253525, "router_z_loss": 0.1678282655775547, "routing_entropy_normalized": 0.835897037247811, "learning_rate": 5.472568979361853e-05, "grad_norm": 0.29108086228370667, "tokens_per_second": 3375927.8175498415, "eta_seconds": 229.84681010246277, "data_wait_seconds": 1.7344951629638672e-05}
+{"step": 28250, "total_steps": 28610, "tokens": 59244544000, "source": "dclm", "phase": 5, "loss": 3.498935490846634, "ce_loss": 3.487337440252304, "logit_z_loss": 23.117897748947144, "router_aux_loss": 1.1379653960466385, "router_z_loss": 0.21834231168031693, "routing_entropy_normalized": 0.8414088564111836, "learning_rate": 5.1824443543202984e-05, "grad_norm": 0.21605584025382996, "tokens_per_second": 3577338.3713546237, "eta_seconds": 211.04369831085205, "data_wait_seconds": 2.1010637283325195e-05}
+{"step": 28260, "total_steps": 28610, "tokens": 59265515520, "source": "cosmopedia_v2", "phase": 5, "loss": 2.7975504100322723, "ce_loss": 2.785767138004303, "logit_z_loss": 26.407479524612427, "router_aux_loss": 1.1368837654590607, "router_z_loss": 0.41442912071943283, "routing_entropy_normalized": 0.8364829848851432, "learning_rate": 4.9000858552895375e-05, "grad_norm": 0.19018103182315826, "tokens_per_second": 3183927.079939361, "eta_seconds": 230.5339229106903, "data_wait_seconds": 1.817941665649414e-05}
+{"step": 28270, "total_steps": 28610, "tokens": 59286487040, "source": "finemath", "phase": 5, "loss": 2.1777670979499817, "ce_loss": 2.166230231523514, "logit_z_loss": 35.03705024719238, "router_aux_loss": 1.1353534907102585, "router_z_loss": 0.18324064277112484, "routing_entropy_normalized": 0.8486954127865337, "learning_rate": 4.625508624298952e-05, "grad_norm": 0.20284980535507202, "tokens_per_second": 4003842.0373605127, "eta_seconds": 178.08686590194702, "data_wait_seconds": 2.065300941467285e-05}
+{"step": 28280, "total_steps": 28610, "tokens": 59307458560, "source": "cosmopedia_v2", "phase": 5, "loss": 2.8057923018932343, "ce_loss": 2.7940456867218018, "logit_z_loss": 26.23194408416748, "router_aux_loss": 1.1350609511137009, "router_z_loss": 0.3959587588906288, "routing_entropy_normalized": 0.8548836148749686, "learning_rate": 4.3587273860921985e-05, "grad_norm": 0.7548964023590088, "tokens_per_second": 3869120.1782415668, "eta_seconds": 178.86757922172546, "data_wait_seconds": 2.047419548034668e-05}
+{"step": 28290, "total_steps": 28610, "tokens": 59328430080, "source": "fineweb_edu", "phase": 5, "loss": 3.4444962441921234, "ce_loss": 3.432883858680725, "logit_z_loss": 22.759189128875732, "router_aux_loss": 1.134047195315361, "router_z_loss": 0.2718684431165457, "routing_entropy_normalized": 0.8584434950743082, "learning_rate": 4.0997564473374104e-05, "grad_norm": 0.2661270499229431, "tokens_per_second": 3734109.833547744, "eta_seconds": 179.718505859375, "data_wait_seconds": 2.1785497665405273e-05}
+{"step": 28300, "total_steps": 28610, "tokens": 59349401600, "source": "cosmopedia_v2", "phase": 5, "loss": 2.7988727390766144, "ce_loss": 2.787124067544937, "logit_z_loss": 26.359848260879517, "router_aux_loss": 1.13512821495533, "router_z_loss": 0.3973729945719242, "routing_entropy_normalized": 0.845530490635873, "learning_rate": 3.848609695860078e-05, "grad_norm": 0.1838477998971939, "tokens_per_second": 3868455.189134277, "eta_seconds": 168.05600380897522, "data_wait_seconds": 2.1070241928100586e-05}
+{"step": 28310, "total_steps": 28610, "tokens": 59370373120, "source": "npset2", "phase": 5, "loss": 1.5900703817605972, "ce_loss": 1.578428015112877, "logit_z_loss": 37.24407148361206, "router_aux_loss": 1.1328914910554886, "router_z_loss": 0.3134901858866215, "routing_entropy_normalized": 0.8001199363371495, "learning_rate": 3.6053005998983254e-05, "grad_norm": 0.30620670318603516, "tokens_per_second": 4159700.851915317, "eta_seconds": 151.2477993965149, "data_wait_seconds": 2.3871660232543945e-05}
+{"step": 28320, "total_steps": 28610, "tokens": 59391344640, "source": "fineweb_edu", "phase": 5, "loss": 3.4307896196842194, "ce_loss": 3.4192073047161102, "logit_z_loss": 22.98045325279236, "router_aux_loss": 1.1324403434991837, "router_z_loss": 0.2578986808657646, "routing_entropy_normalized": 0.8633059340499628, "learning_rate": 3.3698422073805744e-05, "grad_norm": 0.23244938254356384, "tokens_per_second": 4087862.565528613, "eta_seconds": 148.77556920051575, "data_wait_seconds": 1.940131187438965e-05}
+{"step": 28330, "total_steps": 28610, "tokens": 59412316160, "source": "dclm", "phase": 5, "loss": 3.5587405264377594, "ce_loss": 3.5471573173999786, "logit_z_loss": 23.03971004486084, "router_aux_loss": 1.1333892196416855, "router_z_loss": 0.24936968088150024, "routing_entropy_normalized": 0.877980665809458, "learning_rate": 3.1422471452258016e-05, "grad_norm": 0.2661083936691284, "tokens_per_second": 3548980.3011921076, "eta_seconds": 165.45669746398926, "data_wait_seconds": 1.773238182067871e-05}
+{"step": 28340, "total_steps": 28610, "tokens": 59433287680, "source": "finemath", "phase": 5, "loss": 2.161890536546707, "ce_loss": 2.1503761410713196, "logit_z_loss": 35.2562518119812, "router_aux_loss": 1.1331002712249756, "router_z_loss": 0.18342773988842964, "routing_entropy_normalized": 0.8298115933289071, "learning_rate": 2.9225276186664653e-05, "grad_norm": 0.241336390376091, "tokens_per_second": 4307812.275111535, "eta_seconds": 131.44283080101013, "data_wait_seconds": 1.996755599975586e-05}
+{"step": 28350, "total_steps": 28610, "tokens": 59454259200, "source": "dclm", "phase": 5, "loss": 3.603043019771576, "ce_loss": 3.5914676189422607, "logit_z_loss": 22.980189561843872, "router_aux_loss": 1.1349316537380219, "router_z_loss": 0.22611239552497864, "routing_entropy_normalized": 0.8539050499207519, "learning_rate": 2.710695410593994e-05, "grad_norm": 0.28729885816574097, "tokens_per_second": 3930034.079686987, "eta_seconds": 138.74167728424072, "data_wait_seconds": 1.9550323486328125e-05}
+{"step": 28360, "total_steps": 28610, "tokens": 59475230720, "source": "cosmopedia_v2", "phase": 5, "loss": 2.7986084520816803, "ce_loss": 2.786814361810684, "logit_z_loss": 26.43400502204895, "router_aux_loss": 1.134001448750496, "router_z_loss": 0.4540867544710636, "routing_entropy_normalized": 0.863240817780386, "learning_rate": 2.5067618809268278e-05, "grad_norm": 0.3730792999267578, "tokens_per_second": 4191109.8343657176, "eta_seconds": 125.0952661037445, "data_wait_seconds": 2.092123031616211e-05}
+{"step": 28370, "total_steps": 28610, "tokens": 59496202240, "source": "finemath", "phase": 5, "loss": 2.338011145591736, "ce_loss": 2.3264894783496857, "logit_z_loss": 34.09506845474243, "router_aux_loss": 1.1331501007080078, "router_z_loss": 0.1901590581983328, "routing_entropy_normalized": 0.8632945612068849, "learning_rate": 2.3107379660012363e-05, "grad_norm": 0.2013065069913864, "tokens_per_second": 4073869.374125321, "eta_seconds": 123.54752540588379, "data_wait_seconds": 2.0265579223632812e-05}
+{"step": 28380, "total_steps": 28610, "tokens": 59517173760, "source": "cosmopedia_v2", "phase": 5, "loss": 2.8032900989055634, "ce_loss": 2.791520267724991, "logit_z_loss": 26.262795448303223, "router_aux_loss": 1.132930040359497, "router_z_loss": 0.4404784068465233, "routing_entropy_normalized": 0.8773594741207841, "learning_rate": 2.122634177984889e-05, "grad_norm": 0.30049648880958557, "tokens_per_second": 3905102.3210375705, "eta_seconds": 123.51659965515137, "data_wait_seconds": 2.053380012512207e-05}
+{"step": 28390, "total_steps": 28610, "tokens": 59538145280, "source": "fineweb_edu", "phase": 5, "loss": 3.4675089716911316, "ce_loss": 3.455911159515381, "logit_z_loss": 22.65322732925415, "router_aux_loss": 1.1331742405891418, "router_z_loss": 0.26605293340981007, "routing_entropy_normalized": 0.8719832066602838, "learning_rate": 1.942460604313073e-05, "grad_norm": 0.1557585448026657, "tokens_per_second": 4058769.8782141972, "eta_seconds": 113.67321968078613, "data_wait_seconds": 0.05573400855064392}
+{"step": 28400, "total_steps": 28610, "tokens": 59559116800, "source": "cosmopedia_v2", "phase": 5, "loss": 2.8138711154460907, "ce_loss": 2.802109569311142, "logit_z_loss": 26.221176624298096, "router_aux_loss": 1.1324796974658966, "router_z_loss": 0.4367651976644993, "routing_entropy_normalized": 0.8714061886179116, "learning_rate": 1.7702269071477428e-05, "grad_norm": 0.42910581827163696, "tokens_per_second": 4255137.711077464, "eta_seconds": 103.49886417388915, "data_wait_seconds": 2.002716064453125e-05}
+{"step": 28410, "total_steps": 28610, "tokens": 59580088320, "source": "npset2", "phase": 5, "loss": 1.5453701317310333, "ce_loss": 1.5337838977575302, "logit_z_loss": 37.7141489982605, "router_aux_loss": 1.1313526332378387, "router_z_loss": 0.27270414121448994, "routing_entropy_normalized": 0.8199449179025451, "learning_rate": 1.605942322859366e-05, "grad_norm": 0.18959227204322815, "tokens_per_second": 3947136.847038353, "eta_seconds": 106.26193523406982, "data_wait_seconds": 1.8864870071411133e-05}
+{"step": 28420, "total_steps": 28610, "tokens": 59601059840, "source": "fineweb_edu", "phase": 5, "loss": 3.3983368277549744, "ce_loss": 3.3867355585098267, "logit_z_loss": 23.005760669708252, "router_aux_loss": 1.1321180760860443, "router_z_loss": 0.2800303529947996, "routing_entropy_normalized": 0.8709546512488237, "learning_rate": 1.4496156615316391e-05, "grad_norm": 0.19778811931610107, "tokens_per_second": 4325836.830928206, "eta_seconds": 92.11139845848083, "data_wait_seconds": 2.181529998779297e-05}
+{"step": 28430, "total_steps": 28610, "tokens": 59622031360, "source": "dclm", "phase": 5, "loss": 3.557070940732956, "ce_loss": 3.5455488860607147, "logit_z_loss": 22.95970129966736, "router_aux_loss": 1.130581259727478, "router_z_loss": 0.21624510176479816, "routing_entropy_normalized": 0.8740810353545111, "learning_rate": 1.3012553064889632e-05, "grad_norm": 0.2222701758146286, "tokens_per_second": 3577949.2385683064, "eta_seconds": 105.5038332939148, "data_wait_seconds": 1.8805265426635742e-05}
+{"step": 28440, "total_steps": 28610, "tokens": 59643002880, "source": "finemath", "phase": 5, "loss": 2.1909379214048386, "ce_loss": 2.179443433880806, "logit_z_loss": 34.62312126159668, "router_aux_loss": 1.132166638970375, "router_z_loss": 0.17282235994935036, "routing_entropy_normalized": 0.8625835337495573, "learning_rate": 1.160869213846938e-05, "grad_norm": 0.3109655976295471, "tokens_per_second": 3886434.521763516, "eta_seconds": 91.73339676856995, "data_wait_seconds": 1.7642974853515625e-05}
+{"step": 28450, "total_steps": 28610, "tokens": 59663974400, "source": "dclm", "phase": 5, "loss": 3.7093681693077087, "ce_loss": 3.6978437900543213, "logit_z_loss": 22.514865159988403, "router_aux_loss": 1.1314284354448318, "router_z_loss": 0.2101354282349348, "routing_entropy_normalized": 0.8804778157302017, "learning_rate": 1.0284649120856527e-05, "grad_norm": 0.2852003574371338, "tokens_per_second": 4002724.9833053183, "eta_seconds": 83.82897186279297, "data_wait_seconds": 1.6927719116210938e-05}
+{"step": 28460, "total_steps": 28610, "tokens": 59684945920, "source": "cosmopedia_v2", "phase": 5, "loss": 2.7980640828609467, "ce_loss": 2.7863463163375854, "logit_z_loss": 26.447378158569336, "router_aux_loss": 1.13259157538414, "router_z_loss": 0.3918267898261547, "routing_entropy_normalized": 0.8735579486976006, "learning_rate": 9.040495016460094e-06, "grad_norm": 0.2401314228773117, "tokens_per_second": 4194120.808001512, "eta_seconds": 75.00327587127686, "data_wait_seconds": 2.2292137145996094e-05}
+{"step": 28470, "total_steps": 28610, "tokens": 59705917440, "source": "finemath", "phase": 5, "loss": 2.147095575928688, "ce_loss": 2.1356002539396286, "logit_z_loss": 35.11658430099487, "router_aux_loss": 1.131084144115448, "router_z_loss": 0.18446779437363148, "routing_entropy_normalized": 0.849569109301432, "learning_rate": 7.876296545488603e-06, "grad_norm": 0.15857058763504028, "tokens_per_second": 4342030.1966386745, "eta_seconds": 67.61843347549438, "data_wait_seconds": 2.2113323211669922e-05}
+{"step": 28480, "total_steps": 28610, "tokens": 59726888960, "source": "cosmopedia_v2", "phase": 5, "loss": 2.79034361243248, "ce_loss": 2.778613954782486, "logit_z_loss": 26.55735969543457, "router_aux_loss": 1.1315055638551712, "router_z_loss": 0.41459862142801285, "routing_entropy_normalized": 0.8836999015030885, "learning_rate": 6.792116140373117e-06, "grad_norm": 0.8346433043479919, "tokens_per_second": 3836790.1106986864, "eta_seconds": 71.05673027038574, "data_wait_seconds": 2.3066997528076172e-05}
+{"step": 28490, "total_steps": 28610, "tokens": 59747860480, "source": "fineweb_edu", "phase": 5, "loss": 3.419430196285248, "ce_loss": 3.4078318774700165, "logit_z_loss": 22.95578694343567, "router_aux_loss": 1.1318596005439758, "router_z_loss": 0.2797381952404976, "routing_entropy_normalized": 0.8790217252021565, "learning_rate": 5.788011942418237e-06, "grad_norm": 0.19771647453308105, "tokens_per_second": 3427991.0365794655, "eta_seconds": 73.41274738311768, "data_wait_seconds": 0.1669522523880005}
+{"step": 28500, "total_steps": 28610, "tokens": 59768832000, "source": "cosmopedia_v2", "phase": 5, "loss": 2.808714598417282, "ce_loss": 2.7969537675380707, "logit_z_loss": 26.27467703819275, "router_aux_loss": 1.1325212270021439, "router_z_loss": 0.4355529546737671, "routing_entropy_normalized": 0.8767801980461712, "learning_rate": 4.864037798685106e-06, "grad_norm": 0.3214420676231384, "tokens_per_second": 4355744.177018424, "eta_seconds": 52.9614942073822, "data_wait_seconds": 2.1070241928100586e-05}
+{"step": 28510, "total_steps": 28610, "tokens": 59789803520, "source": "npset2", "phase": 5, "loss": 1.5507932305335999, "ce_loss": 1.5392421782016754, "logit_z_loss": 37.51888418197632, "router_aux_loss": 1.1288949251174927, "router_z_loss": 0.262073477730155, "routing_entropy_normalized": 0.8287835663251537, "learning_rate": 4.0202432591025515e-06, "grad_norm": 0.235471710562706, "tokens_per_second": 4154930.908569766, "eta_seconds": 50.473811626434326, "data_wait_seconds": 2.2590160369873047e-05}
+{"step": 28520, "total_steps": 28610, "tokens": 59810775040, "source": "fineweb_edu", "phase": 5, "loss": 3.4657942354679108, "ce_loss": 3.454213470220566, "logit_z_loss": 22.91712236404419, "router_aux_loss": 1.1308083683252335, "router_z_loss": 0.27263848669826984, "routing_entropy_normalized": 0.8812989727866994, "learning_rate": 3.2566735738112084e-06, "grad_norm": 0.2372068464756012, "tokens_per_second": 4119704.061784791, "eta_seconds": 45.81486368179321, "data_wait_seconds": 2.1070241928100586e-05}
+{"step": 28530, "total_steps": 28610, "tokens": 59831746560, "source": "dclm", "phase": 5, "loss": 3.616079032421112, "ce_loss": 3.6045749187469482, "logit_z_loss": 22.817455768585205, "router_aux_loss": 1.1303915530443192, "router_z_loss": 0.200216518715024, "routing_entropy_normalized": 0.876950894743613, "learning_rate": 2.5733696907359604e-06, "grad_norm": 0.16323983669281006, "tokens_per_second": 3744772.1354980124, "eta_seconds": 44.801700592041016, "data_wait_seconds": 1.8477439880371094e-05}
+{"step": 28540, "total_steps": 28610, "tokens": 59852718080, "source": "finemath", "phase": 5, "loss": 1.9722069054841995, "ce_loss": 1.9607226699590683, "logit_z_loss": 35.53999662399292, "router_aux_loss": 1.1305185109376907, "router_z_loss": 0.17905435152351856, "routing_entropy_normalized": 0.8603118755691536, "learning_rate": 1.9703682533901977e-06, "grad_norm": 0.14570476114749908, "tokens_per_second": 4053083.8809498916, "eta_seconds": 36.21949219703674, "data_wait_seconds": 1.6570091247558594e-05}
+{"step": 28550, "total_steps": 28610, "tokens": 59873689600, "source": "dclm", "phase": 5, "loss": 3.6269178688526154, "ce_loss": 3.6153849959373474, "logit_z_loss": 22.700026512145996, "router_aux_loss": 1.130830094218254, "router_z_loss": 0.2246313076466322, "routing_entropy_normalized": 0.883886407410607, "learning_rate": 1.4477015989110553e-06, "grad_norm": 0.17170587182044983, "tokens_per_second": 4308296.087019215, "eta_seconds": 29.206237792968754, "data_wait_seconds": 1.8924474716186523e-05}
+{"step": 28560, "total_steps": 28610, "tokens": 59894661120, "source": "cosmopedia_v2", "phase": 5, "loss": 2.790831744670868, "ce_loss": 2.7791003584861755, "logit_z_loss": 26.612406969070435, "router_aux_loss": 1.1315158903598785, "router_z_loss": 0.4162539392709732, "routing_entropy_normalized": 0.8776584520264326, "learning_rate": 1.0053977563251326e-06, "grad_norm": 0.1968858540058136, "tokens_per_second": 3863592.649082323, "eta_seconds": 27.139921188354492, "data_wait_seconds": 2.104043960571289e-05}
+{"step": 28570, "total_steps": 28610, "tokens": 59915632640, "source": "finemath", "phase": 5, "loss": 2.1863881051540375, "ce_loss": 2.1747566610574722, "logit_z_loss": 35.00465106964111, "router_aux_loss": 1.1303027719259262, "router_z_loss": 0.3284557834267616, "routing_entropy_normalized": 0.8679049635705, "learning_rate": 6.434804450450305e-07, "grad_norm": 0.20022663474082947, "tokens_per_second": 3654851.925509454, "eta_seconds": 22.951977729797363, "data_wait_seconds": 0.12939512729644775}
+{"step": 28580, "total_steps": 28610, "tokens": 59936604160, "source": "cosmopedia_v2", "phase": 5, "loss": 2.7991241812705994, "ce_loss": 2.787378579378128, "logit_z_loss": 26.49033236503601, "router_aux_loss": 1.1313853114843369, "router_z_loss": 0.43172862753272057, "routing_entropy_normalized": 0.8733043988490443, "learning_rate": 3.619690735978676e-07, "grad_norm": 0.24607665836811066, "tokens_per_second": 4320361.44392654, "eta_seconds": 14.562337160110474, "data_wait_seconds": 2.1576881408691406e-05}
+{"step": 28590, "total_steps": 28610, "tokens": 59957575680, "source": "fineweb_edu", "phase": 5, "loss": 3.399412751197815, "ce_loss": 3.387845039367676, "logit_z_loss": 23.18182921409607, "router_aux_loss": 1.1305192410945892, "router_z_loss": 0.26251762360334396, "routing_entropy_normalized": 0.8889308333070203, "learning_rate": 1.6087873858411373e-07, "grad_norm": 0.14011861383914948, "tokens_per_second": 4149526.2000738285, "eta_seconds": 10.107910633087158, "data_wait_seconds": 2.2113323211669922e-05}
+{"step": 28600, "total_steps": 28610, "tokens": 59978547200, "source": "cosmopedia_v2", "phase": 5, "loss": 2.774931013584137, "ce_loss": 2.7632090747356415, "logit_z_loss": 26.57383894920349, "router_aux_loss": 1.1307422965765, "router_z_loss": 0.41450344026088715, "routing_entropy_normalized": 0.8807967830612493, "learning_rate": 4.022022386823698e-08, "grad_norm": 0.23850508034229279, "tokens_per_second": 3769684.5747681344, "eta_seconds": 5.563202857971191, "data_wait_seconds": 2.0742416381835938e-05}
+{"step": 28610, "total_steps": 28610, "tokens": 59999518720, "source": "npset2", "phase": 5, "loss": 1.745762512087822, "ce_loss": 1.7341523617506027, "logit_z_loss": 36.487751960754395, "router_aux_loss": 1.1288997828960419, "router_z_loss": 0.3211454078555107, "routing_entropy_normalized": 0.8315624889632758, "learning_rate": 0.0, "grad_norm": 0.25187817215919495, "tokens_per_second": 4184896.9457226605, "eta_seconds": 0.0, "data_wait_seconds": 2.1636486053466797e-05}