Upload SLMoE 5% checkpoint at 3,001,024,512 tokens
Browse files- README.md +77 -0
- checkpoint_metadata.json +84 -0
- config.json +52 -0
- generation_config.json +7 -0
- model.safetensors +3 -0
- special_tokens_map.json +6 -0
- tokenizer.json +0 -0
- tokenizer_config.json +8 -0
- training_metrics.jsonl +144 -0
README.md
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
language:
|
| 3 |
+
- en
|
| 4 |
+
library_name: transformers
|
| 5 |
+
pipeline_tag: text-generation
|
| 6 |
+
datasets:
|
| 7 |
+
- epfml/FineWeb-HQ
|
| 8 |
+
- HuggingFaceFW/fineweb-edu
|
| 9 |
+
- mlfoundations/dclm-baseline-1.0
|
| 10 |
+
- HuggingFaceTB/smollm-corpus
|
| 11 |
+
- HuggingFaceTB/finemath
|
| 12 |
+
- AxiomicLabs/NPset-2-Python-Edu
|
| 13 |
+
tags:
|
| 14 |
+
- causal-lm
|
| 15 |
+
- base-model
|
| 16 |
+
- mixture-of-experts
|
| 17 |
+
- sequence-routing
|
| 18 |
+
- custom-code
|
| 19 |
+
- trust-remote-code
|
| 20 |
+
---
|
| 21 |
+
|
| 22 |
+
# SLMoE Test - 5%
|
| 23 |
+
|
| 24 |
+
This is the **5% checkpoint** of an experimental sequence-routed base
|
| 25 |
+
language model. It is not instruction tuned.
|
| 26 |
+
|
| 27 |
+
## Architecture
|
| 28 |
+
|
| 29 |
+
| Field | Value |
|
| 30 |
+
|---|---:|
|
| 31 |
+
| Total parameters | 25,449,472 |
|
| 32 |
+
| Active parameters per sequence | 7,902,208 |
|
| 33 |
+
| Layers / hidden size | 8 / 256 |
|
| 34 |
+
| Query / KV heads | 8 / 2 |
|
| 35 |
+
| Head dimension | 32 |
|
| 36 |
+
| Expert paths | 64 total / 13 active |
|
| 37 |
+
| Expert intermediate size | 56 |
|
| 38 |
+
| Context | 4,096 |
|
| 39 |
+
| Vocabulary | 8,192, tied |
|
| 40 |
+
|
| 41 |
+
The router reads only the first 32 valid tokens and makes one global top-13
|
| 42 |
+
decision. Those expert identities are reused in every layer and stored in the
|
| 43 |
+
KV cache for the complete generated response. Routing does not change per
|
| 44 |
+
token. During pretraining, one packed 4,096-token sequence is the routing unit,
|
| 45 |
+
and loss on its routing prefix is masked to prevent future-token leakage.
|
| 46 |
+
|
| 47 |
+
## Training
|
| 48 |
+
|
| 49 |
+
| Field | Value |
|
| 50 |
+
|---|---:|
|
| 51 |
+
| Progress | 5% |
|
| 52 |
+
| Tokens seen | 3,001,024,512 |
|
| 53 |
+
| Target tokens | 60,000,000,000 |
|
| 54 |
+
| Hardware | 4 x NVIDIA H200 |
|
| 55 |
+
| Optimizer | Fused AdamW |
|
| 56 |
+
| Peak LR | 0.003 |
|
| 57 |
+
| Betas | 0.9, 0.95 |
|
| 58 |
+
| Weight decay | 0.1 |
|
| 59 |
+
| Precision | bfloat16 autocast |
|
| 60 |
+
|
| 61 |
+
| Token range | FineWeb-HQ | FineWeb-Edu | DCLM | Cosmopedia v2 | FineMath | NPSet-2 |
|
| 62 |
+
|---|---:|---:|---:|---:|---:|---:|
|
| 63 |
+
| 0.00B-12.00B | 35% | 30% | 20% | 10% | 4% | 1% |
|
| 64 |
+
| 12.00B-24.00B | 32% | 30% | 18% | 11% | 7% | 2% |
|
| 65 |
+
| 24.00B-39.00B | 28% | 29% | 16% | 13% | 11% | 3% |
|
| 66 |
+
| 39.00B-51.00B | 24% | 27% | 13% | 14% | 18% | 4% |
|
| 67 |
+
| 51.00B-60.00B | 20% | 24% | 9% | 15% | 26% | 6% |
|
| 68 |
+
|
| 69 |
+
## Usage
|
| 70 |
+
|
| 71 |
+
```python
|
| 72 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 73 |
+
|
| 74 |
+
model_id = "Banaxi-Tech/slmoe-test"
|
| 75 |
+
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
|
| 76 |
+
model = AutoModelForCausalLM.from_pretrained(model_id, trust_remote_code=True)
|
| 77 |
+
```
|
checkpoint_metadata.json
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"parameters": 25449472,
|
| 3 |
+
"active_parameters": 7902208,
|
| 4 |
+
"architecture": {
|
| 5 |
+
"vocab_size": 8192,
|
| 6 |
+
"hidden_size": 256,
|
| 7 |
+
"num_hidden_layers": 8,
|
| 8 |
+
"num_attention_heads": 8,
|
| 9 |
+
"num_key_value_heads": 2,
|
| 10 |
+
"head_dim": 32,
|
| 11 |
+
"num_experts": 64,
|
| 12 |
+
"num_experts_per_sequence": 13,
|
| 13 |
+
"expert_intermediate_size": 56,
|
| 14 |
+
"router_prefix_length": 32,
|
| 15 |
+
"router_jitter_noise": 0.01,
|
| 16 |
+
"router_aux_loss_coeff": 0.01,
|
| 17 |
+
"router_z_loss_coeff": 0.001,
|
| 18 |
+
"expert_output_scale": 3.605551275463989,
|
| 19 |
+
"max_position_embeddings": 4096,
|
| 20 |
+
"rope_theta": 100000.0,
|
| 21 |
+
"rms_norm_eps": 1e-06,
|
| 22 |
+
"initializer_range": 0.02,
|
| 23 |
+
"use_cache": true,
|
| 24 |
+
"transformers_version": "5.15.0",
|
| 25 |
+
"architectures": null,
|
| 26 |
+
"output_hidden_states": false,
|
| 27 |
+
"return_dict": true,
|
| 28 |
+
"dtype": null,
|
| 29 |
+
"chunk_size_feed_forward": 0,
|
| 30 |
+
"is_encoder_decoder": false,
|
| 31 |
+
"id2label": {
|
| 32 |
+
"0": "LABEL_0",
|
| 33 |
+
"1": "LABEL_1"
|
| 34 |
+
},
|
| 35 |
+
"label2id": {
|
| 36 |
+
"LABEL_0": 0,
|
| 37 |
+
"LABEL_1": 1
|
| 38 |
+
},
|
| 39 |
+
"problem_type": null,
|
| 40 |
+
"_name_or_path": "",
|
| 41 |
+
"tie_word_embeddings": true,
|
| 42 |
+
"bos_token_id": 1,
|
| 43 |
+
"eos_token_id": 2,
|
| 44 |
+
"pad_token_id": 0,
|
| 45 |
+
"unk_token_id": 3,
|
| 46 |
+
"model_type": "slmoe",
|
| 47 |
+
"output_attentions": false
|
| 48 |
+
},
|
| 49 |
+
"training_percent": 5,
|
| 50 |
+
"step": 1431,
|
| 51 |
+
"total_steps": 28610,
|
| 52 |
+
"tokens_seen": 3001024512,
|
| 53 |
+
"target_tokens": 60000000000,
|
| 54 |
+
"trained_token_target": 59999518720,
|
| 55 |
+
"tokens_per_step": 2097152,
|
| 56 |
+
"tokens_per_micro_batch": 1048576,
|
| 57 |
+
"local_batch": 64,
|
| 58 |
+
"effective_global_batch": 512,
|
| 59 |
+
"gradient_accumulation_steps": 2,
|
| 60 |
+
"world_size": 4,
|
| 61 |
+
"gpu_name": "NVIDIA H200",
|
| 62 |
+
"optimizer": "AdamW",
|
| 63 |
+
"peak_lr": 0.003,
|
| 64 |
+
"architecture_revision": "bd0ade7689958997e1889c641a782a69c0bea918",
|
| 65 |
+
"tokenizer_repo": "BananaMind/BananaMind-2-Mini",
|
| 66 |
+
"tokenizer_revision": "6400d0a6dcbe52f3c725291cb97c079c591c05a4",
|
| 67 |
+
"dataset_revisions": {
|
| 68 |
+
"fineweb_hq": "e58199cdd52438d94405df1a4d8630cc5f13bf84",
|
| 69 |
+
"fineweb_edu": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
|
| 70 |
+
"dclm": "a3b142c183aebe5af344955ae20836eb34dcf69b",
|
| 71 |
+
"cosmopedia_v2": "3ba9d605774198c5868892d7a8deda78031a781f",
|
| 72 |
+
"finemath": "e92b25a616738fe95dc186b64dfb19f9c8525594",
|
| 73 |
+
"npset2": "222b911cd9119624dd65ab084d69542f6e04631e"
|
| 74 |
+
},
|
| 75 |
+
"source_tokens": {
|
| 76 |
+
"fineweb_hq": 1050673152,
|
| 77 |
+
"fineweb_edu": 899678208,
|
| 78 |
+
"dclm": 599785472,
|
| 79 |
+
"cosmopedia_v2": 299892736,
|
| 80 |
+
"finemath": 119537664,
|
| 81 |
+
"npset2": 31457280
|
| 82 |
+
},
|
| 83 |
+
"elapsed_seconds": 905.3876543045044
|
| 84 |
+
}
|
config.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"vocab_size": 8192,
|
| 3 |
+
"hidden_size": 256,
|
| 4 |
+
"num_hidden_layers": 8,
|
| 5 |
+
"num_attention_heads": 8,
|
| 6 |
+
"num_key_value_heads": 2,
|
| 7 |
+
"head_dim": 32,
|
| 8 |
+
"num_experts": 64,
|
| 9 |
+
"num_experts_per_sequence": 13,
|
| 10 |
+
"expert_intermediate_size": 56,
|
| 11 |
+
"router_prefix_length": 32,
|
| 12 |
+
"router_jitter_noise": 0.01,
|
| 13 |
+
"router_aux_loss_coeff": 0.01,
|
| 14 |
+
"router_z_loss_coeff": 0.001,
|
| 15 |
+
"expert_output_scale": 3.605551275463989,
|
| 16 |
+
"max_position_embeddings": 4096,
|
| 17 |
+
"rope_theta": 100000.0,
|
| 18 |
+
"rms_norm_eps": 1e-06,
|
| 19 |
+
"initializer_range": 0.02,
|
| 20 |
+
"use_cache": true,
|
| 21 |
+
"transformers_version": "5.15.0",
|
| 22 |
+
"architectures": [
|
| 23 |
+
"SLMoEForCausalLM"
|
| 24 |
+
],
|
| 25 |
+
"output_hidden_states": false,
|
| 26 |
+
"return_dict": true,
|
| 27 |
+
"dtype": null,
|
| 28 |
+
"chunk_size_feed_forward": 0,
|
| 29 |
+
"is_encoder_decoder": false,
|
| 30 |
+
"id2label": {
|
| 31 |
+
"0": "LABEL_0",
|
| 32 |
+
"1": "LABEL_1"
|
| 33 |
+
},
|
| 34 |
+
"label2id": {
|
| 35 |
+
"LABEL_0": 0,
|
| 36 |
+
"LABEL_1": 1
|
| 37 |
+
},
|
| 38 |
+
"problem_type": null,
|
| 39 |
+
"_name_or_path": "Banaxi-Tech/slmoe-test",
|
| 40 |
+
"tie_word_embeddings": true,
|
| 41 |
+
"bos_token_id": 1,
|
| 42 |
+
"eos_token_id": 2,
|
| 43 |
+
"pad_token_id": 0,
|
| 44 |
+
"unk_token_id": 3,
|
| 45 |
+
"model_type": "slmoe",
|
| 46 |
+
"output_attentions": false,
|
| 47 |
+
"auto_map": {
|
| 48 |
+
"AutoConfig": "configuration_slmoe.SLMoEConfig",
|
| 49 |
+
"AutoModelForCausalLM": "modeling_slmoe.SLMoEForCausalLM"
|
| 50 |
+
},
|
| 51 |
+
"torch_dtype": "float32"
|
| 52 |
+
}
|
generation_config.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 1,
|
| 4 |
+
"eos_token_id": 2,
|
| 5 |
+
"pad_token_id": 0,
|
| 6 |
+
"transformers_version": "5"
|
| 7 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:be863d7d7add4f5b771a3adfadb63d57befaf4eddeb2d538735bd77a49ebaed6
|
| 3 |
+
size 110196488
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"pad_token": "<|pad|>",
|
| 3 |
+
"bos_token": "<|bos|>",
|
| 4 |
+
"eos_token": "<|eos|>",
|
| 5 |
+
"unk_token": "<|unk|>"
|
| 6 |
+
}
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 3 |
+
"model_max_length": 4096,
|
| 4 |
+
"pad_token": "<|pad|>",
|
| 5 |
+
"bos_token": "<|bos|>",
|
| 6 |
+
"eos_token": "<|eos|>",
|
| 7 |
+
"unk_token": "<|unk|>"
|
| 8 |
+
}
|
training_metrics.jsonl
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"step": 1, "total_steps": 28610, "tokens": 2097152, "source": "fineweb_hq", "phase": 1, "loss": 9.073994517326355, "ce_loss": 9.046570062637329, "logit_z_loss": 0.10562420077621937, "router_aux_loss": 1.0150056183338165, "router_z_loss": 17.263988971710205, "routing_entropy_normalized": 0.9620906325822177, "learning_rate": 3e-06, "grad_norm": 0.36904314160346985, "tokens_per_second": 41938.777033319384, "eta_seconds": 1430595.40149045, "data_wait_seconds": 5.502631962299347}
|
| 2 |
+
{"step": 10, "total_steps": 28610, "tokens": 20971520, "source": "fineweb_hq", "phase": 1, "loss": 9.055508732795715, "ce_loss": 9.02808964252472, "logit_z_loss": 0.10522722825407982, "router_aux_loss": 1.0159947872161865, "router_z_loss": 17.248795747756958, "routing_entropy_normalized": 0.9600620670177143, "learning_rate": 2.9999999999999997e-05, "grad_norm": 0.3937816321849823, "tokens_per_second": 589150.7098755764, "eta_seconds": 101805.10045157537, "data_wait_seconds": 2.0503997802734375e-05}
|
| 3 |
+
{"step": 20, "total_steps": 28610, "tokens": 41943040, "source": "fineweb_hq", "phase": 1, "loss": 8.964744091033936, "ce_loss": 8.937372088432312, "logit_z_loss": 0.10431208368390799, "router_aux_loss": 1.0176522433757782, "router_z_loss": 17.185110092163086, "routing_entropy_normalized": 0.9568416245510577, "learning_rate": 5.9999999999999995e-05, "grad_norm": 0.6630691885948181, "tokens_per_second": 2107568.5835980666, "eta_seconds": 28448.694930553436, "data_wait_seconds": 2.3573637008666992e-05}
|
| 4 |
+
{"step": 30, "total_steps": 28610, "tokens": 62914560, "source": "fineweb_hq", "phase": 1, "loss": 8.579839587211609, "ce_loss": 8.552549600601196, "logit_z_loss": 0.11818500328809023, "router_aux_loss": 1.0216142386198044, "router_z_loss": 17.062674045562744, "routing_entropy_normalized": 0.9474573234194218, "learning_rate": 8.999999999999999e-05, "grad_norm": 1.15729820728302, "tokens_per_second": 4129725.0589066464, "eta_seconds": 14513.461139678955, "data_wait_seconds": 2.142786979675293e-05}
|
| 5 |
+
{"step": 40, "total_steps": 28610, "tokens": 83886080, "source": "fineweb_hq", "phase": 1, "loss": 8.270026326179504, "ce_loss": 8.242919445037842, "logit_z_loss": 0.2141006626188755, "router_aux_loss": 1.0286758691072464, "router_z_loss": 16.798901319503784, "routing_entropy_normalized": 0.9313903150915323, "learning_rate": 0.00011999999999999999, "grad_norm": 1.103349208831787, "tokens_per_second": 1107003.578922341, "eta_seconds": 54124.154411792755, "data_wait_seconds": 1.704692840576172e-05}
|
| 6 |
+
{"step": 50, "total_steps": 28610, "tokens": 104857600, "source": "fineweb_edu", "phase": 1, "loss": 7.979007363319397, "ce_loss": 7.9522459506988525, "logit_z_loss": 0.4787629134953022, "router_aux_loss": 1.0292921960353851, "router_z_loss": 16.42088222503662, "routing_entropy_normalized": 0.9325584503163254, "learning_rate": 0.00015, "grad_norm": 1.0911495685577393, "tokens_per_second": 4101685.7685916405, "eta_seconds": 14602.44994354248, "data_wait_seconds": 1.9311904907226562e-05}
|
| 7 |
+
{"step": 60, "total_steps": 28610, "tokens": 125829120, "source": "fineweb_hq", "phase": 1, "loss": 7.6750635504722595, "ce_loss": 7.648828864097595, "logit_z_loss": 1.0512438714504242, "router_aux_loss": 1.0295023769140244, "router_z_loss": 15.83453106880188, "routing_entropy_normalized": 0.9327803314012314, "learning_rate": 0.00017999999999999998, "grad_norm": 0.9790343642234802, "tokens_per_second": 4202982.720652348, "eta_seconds": 14245.523614883421, "data_wait_seconds": 1.952052116394043e-05}
|
| 8 |
+
{"step": 70, "total_steps": 28610, "tokens": 146800640, "source": "fineweb_edu", "phase": 1, "loss": 7.454107284545898, "ce_loss": 7.428394019603729, "logit_z_loss": 1.92060786485672, "router_aux_loss": 1.0272448360919952, "router_z_loss": 15.248664021492004, "routing_entropy_normalized": 0.9381905292100782, "learning_rate": 0.00020999999999999998, "grad_norm": 0.8559066653251648, "tokens_per_second": 3371260.2574310955, "eta_seconds": 17753.811189174652, "data_wait_seconds": 1.901388168334961e-05}
|
| 9 |
+
{"step": 80, "total_steps": 28610, "tokens": 167772160, "source": "fineweb_hq", "phase": 1, "loss": 7.25084000825882, "ce_loss": 7.225758194923401, "logit_z_loss": 2.8927885591983795, "router_aux_loss": 1.030070498585701, "router_z_loss": 14.491877794265747, "routing_entropy_normalized": 0.9341402720953963, "learning_rate": 0.00023999999999999998, "grad_norm": 0.5455807447433472, "tokens_per_second": 4119744.195539464, "eta_seconds": 14523.170303821564, "data_wait_seconds": 1.946091651916504e-05}
|
| 10 |
+
{"step": 90, "total_steps": 28610, "tokens": 188743680, "source": "fineweb_edu", "phase": 1, "loss": 7.154139280319214, "ce_loss": 7.129468023777008, "logit_z_loss": 3.583126336336136, "router_aux_loss": 1.0286142379045486, "router_z_loss": 14.02677857875824, "routing_entropy_normalized": 0.9412246630090678, "learning_rate": 0.00027, "grad_norm": 0.3389282524585724, "tokens_per_second": 3321389.154513384, "eta_seconds": 18007.75887966156, "data_wait_seconds": 2.327561378479004e-05}
|
| 11 |
+
{"step": 100, "total_steps": 28610, "tokens": 209715200, "source": "fineweb_hq", "phase": 1, "loss": 6.977815926074982, "ce_loss": 6.953558504581451, "logit_z_loss": 3.842698246240616, "router_aux_loss": 1.0332032442092896, "router_z_loss": 13.541024327278137, "routing_entropy_normalized": 0.9356706655893746, "learning_rate": 0.0003, "grad_norm": 0.26512834429740906, "tokens_per_second": 3789044.883556809, "eta_seconds": 15779.65037560463, "data_wait_seconds": 1.996755599975586e-05}
|
| 12 |
+
{"step": 110, "total_steps": 28610, "tokens": 230686720, "source": "fineweb_hq", "phase": 1, "loss": 6.898625373840332, "ce_loss": 6.8747740387916565, "logit_z_loss": 4.094127714633942, "router_aux_loss": 1.0331053882837296, "router_z_loss": 13.110895991325378, "routing_entropy_normalized": 0.9411351697257977, "learning_rate": 0.00033, "grad_norm": 0.2597975432872772, "tokens_per_second": 3735055.8041023863, "eta_seconds": 16002.125573158264, "data_wait_seconds": 2.0772218704223633e-05}
|
| 13 |
+
{"step": 120, "total_steps": 28610, "tokens": 251658240, "source": "fineweb_hq", "phase": 1, "loss": 6.798978626728058, "ce_loss": 6.775203168392181, "logit_z_loss": 4.276031851768494, "router_aux_loss": 1.0313971191644669, "router_z_loss": 13.033982276916504, "routing_entropy_normalized": 0.9455665700199555, "learning_rate": 0.00035999999999999997, "grad_norm": 0.26467564702033997, "tokens_per_second": 3665904.2181408326, "eta_seconds": 16298.260108470917, "data_wait_seconds": 1.9162893295288086e-05}
|
| 14 |
+
{"step": 130, "total_steps": 28610, "tokens": 272629760, "source": "fineweb_hq", "phase": 1, "loss": 6.614393591880798, "ce_loss": 6.591010928153992, "logit_z_loss": 4.078372240066528, "router_aux_loss": 1.0381052941083908, "router_z_loss": 12.593927025794983, "routing_entropy_normalized": 0.937294403890337, "learning_rate": 0.00039, "grad_norm": 0.27912798523902893, "tokens_per_second": 3925328.4324998586, "eta_seconds": 15215.768562316895, "data_wait_seconds": 1.958012580871582e-05}
|
| 15 |
+
{"step": 140, "total_steps": 28610, "tokens": 293601280, "source": "fineweb_hq", "phase": 1, "loss": 6.468375325202942, "ce_loss": 6.444966495037079, "logit_z_loss": 4.336017906665802, "router_aux_loss": 1.033752515912056, "router_z_loss": 12.637640118598938, "routing_entropy_normalized": 0.9428963136152974, "learning_rate": 0.00041999999999999996, "grad_norm": 0.2304970920085907, "tokens_per_second": 3953973.20982587, "eta_seconds": 15100.233175992966, "data_wait_seconds": 2.0205974578857422e-05}
|
| 16 |
+
{"step": 150, "total_steps": 28610, "tokens": 314572800, "source": "fineweb_edu", "phase": 1, "loss": 6.38175904750824, "ce_loss": 6.358521997928619, "logit_z_loss": 4.819796442985535, "router_aux_loss": 1.032863363623619, "router_z_loss": 12.426431655883789, "routing_entropy_normalized": 0.9432915428506657, "learning_rate": 0.00045, "grad_norm": 0.3845673203468323, "tokens_per_second": 3776318.5268701157, "eta_seconds": 15805.061330318451, "data_wait_seconds": 1.7195940017700195e-05}
|
| 17 |
+
{"step": 160, "total_steps": 28610, "tokens": 335544320, "source": "fineweb_hq", "phase": 1, "loss": 6.226489305496216, "ce_loss": 6.203530251979828, "logit_z_loss": 4.996673285961151, "router_aux_loss": 1.0317778289318085, "router_z_loss": 12.14176607131958, "routing_entropy_normalized": 0.9463213114382736, "learning_rate": 0.00047999999999999996, "grad_norm": 0.24340784549713135, "tokens_per_second": 3965274.8300431576, "eta_seconds": 15046.61768913269, "data_wait_seconds": 1.6570091247558594e-05}
|
| 18 |
+
{"step": 170, "total_steps": 28610, "tokens": 356515840, "source": "fineweb_edu", "phase": 1, "loss": 6.168450474739075, "ce_loss": 6.145536780357361, "logit_z_loss": 4.875605285167694, "router_aux_loss": 1.0287566035985947, "router_z_loss": 12.138543725013733, "routing_entropy_normalized": 0.9464020197920747, "learning_rate": 0.00051, "grad_norm": 0.22221329808235168, "tokens_per_second": 3201967.8943166654, "eta_seconds": 18626.98342037201, "data_wait_seconds": 2.2351741790771484e-05}
|
| 19 |
+
{"step": 180, "total_steps": 28610, "tokens": 377487360, "source": "fineweb_hq", "phase": 1, "loss": 6.026467323303223, "ce_loss": 6.00395667552948, "logit_z_loss": 4.965599536895752, "router_aux_loss": 1.0341257005929947, "router_z_loss": 11.67285430431366, "routing_entropy_normalized": 0.936981342961308, "learning_rate": 0.00054, "grad_norm": 0.3171170949935913, "tokens_per_second": 3356365.6935463715, "eta_seconds": 17763.86627793312, "data_wait_seconds": 1.913309097290039e-05}
|
| 20 |
+
{"step": 190, "total_steps": 28610, "tokens": 398458880, "source": "fineweb_edu", "phase": 1, "loss": 5.9901896715164185, "ce_loss": 5.967985987663269, "logit_z_loss": 5.4937387108802795, "router_aux_loss": 1.0335818380117416, "router_z_loss": 11.31857442855835, "routing_entropy_normalized": 0.9365698092361668, "learning_rate": 0.0005700000000000001, "grad_norm": 0.433395653963089, "tokens_per_second": 3735590.6813575695, "eta_seconds": 15954.922507286072, "data_wait_seconds": 2.130866050720215e-05}
|
| 21 |
+
{"step": 200, "total_steps": 28610, "tokens": 419430400, "source": "fineweb_hq", "phase": 1, "loss": 5.849806308746338, "ce_loss": 5.828191161155701, "logit_z_loss": 5.523493230342865, "router_aux_loss": 1.0332033932209015, "router_z_loss": 10.73071539402008, "routing_entropy_normalized": 0.9392291897953079, "learning_rate": 0.0006, "grad_norm": 0.26707807183265686, "tokens_per_second": 3512841.2527675414, "eta_seconds": 16960.654932260513, "data_wait_seconds": 2.0623207092285156e-05}
|
| 22 |
+
{"step": 210, "total_steps": 28610, "tokens": 440401920, "source": "fineweb_hq", "phase": 1, "loss": 5.789299488067627, "ce_loss": 5.7680951952934265, "logit_z_loss": 5.37375545501709, "router_aux_loss": 1.03995081782341, "router_z_loss": 10.26725447177887, "routing_entropy_normalized": 0.9331191077656645, "learning_rate": 0.00063, "grad_norm": 0.3277837336063385, "tokens_per_second": 4326161.923050119, "eta_seconds": 13767.195463180542, "data_wait_seconds": 2.4527311325073242e-05}
|
| 23 |
+
{"step": 220, "total_steps": 28610, "tokens": 461373440, "source": "fineweb_hq", "phase": 1, "loss": 5.754351615905762, "ce_loss": 5.733106255531311, "logit_z_loss": 5.4533281326293945, "router_aux_loss": 1.0382436513900757, "router_z_loss": 10.317654967308044, "routing_entropy_normalized": 0.9395890160509555, "learning_rate": 0.00066, "grad_norm": 0.3422394394874573, "tokens_per_second": 3484067.579029589, "eta_seconds": 17088.688416481018, "data_wait_seconds": 2.0176172256469727e-05}
|
| 24 |
+
{"step": 230, "total_steps": 28610, "tokens": 482344960, "source": "fineweb_hq", "phase": 1, "loss": 5.676178991794586, "ce_loss": 5.655082106590271, "logit_z_loss": 5.8926796317100525, "router_aux_loss": 1.0480426847934723, "router_z_loss": 10.027278900146484, "routing_entropy_normalized": 0.9332907984313041, "learning_rate": 0.0006900000000000001, "grad_norm": 0.2869788408279419, "tokens_per_second": 3749454.125665829, "eta_seconds": 15873.556993961334, "data_wait_seconds": 1.9490718841552734e-05}
|
| 25 |
+
{"step": 240, "total_steps": 28610, "tokens": 503316480, "source": "fineweb_hq", "phase": 1, "loss": 5.61147153377533, "ce_loss": 5.590435147285461, "logit_z_loss": 6.474088132381439, "router_aux_loss": 1.053972065448761, "router_z_loss": 9.849324941635132, "routing_entropy_normalized": 0.9290603602817278, "learning_rate": 0.0007199999999999999, "grad_norm": 0.33774682879447937, "tokens_per_second": 3825417.806882809, "eta_seconds": 15552.863829135895, "data_wait_seconds": 2.0444393157958984e-05}
|
| 26 |
+
{"step": 250, "total_steps": 28610, "tokens": 524288000, "source": "fineweb_edu", "phase": 1, "loss": 5.610336422920227, "ce_loss": 5.589332938194275, "logit_z_loss": 6.556094706058502, "router_aux_loss": 1.0614578574895859, "router_z_loss": 9.733118772506714, "routing_entropy_normalized": 0.9243547477714621, "learning_rate": 0.00075, "grad_norm": 0.3457306921482086, "tokens_per_second": 3661759.3486100407, "eta_seconds": 16242.255445480347, "data_wait_seconds": 1.856684684753418e-05}
|
| 27 |
+
{"step": 260, "total_steps": 28610, "tokens": 545259520, "source": "fineweb_hq", "phase": 1, "loss": 5.5172039270401, "ce_loss": 5.496480822563171, "logit_z_loss": 6.524185061454773, "router_aux_loss": 1.0688736885786057, "router_z_loss": 9.382022023200989, "routing_entropy_normalized": 0.9318456118932588, "learning_rate": 0.00078, "grad_norm": 0.22909705340862274, "tokens_per_second": 3765612.1404037904, "eta_seconds": 15788.736859560013, "data_wait_seconds": 2.294778823852539e-05}
|
| 28 |
+
{"step": 270, "total_steps": 28610, "tokens": 566231040, "source": "fineweb_edu", "phase": 1, "loss": 5.5132521986961365, "ce_loss": 5.492914617061615, "logit_z_loss": 6.372886538505554, "router_aux_loss": 1.0843495279550552, "router_z_loss": 8.856839776039124, "routing_entropy_normalized": 0.9234213649311608, "learning_rate": 0.0008100000000000001, "grad_norm": 0.40789803862571716, "tokens_per_second": 3457978.330706527, "eta_seconds": 17187.293266773224, "data_wait_seconds": 2.03549861907959e-05}
|
| 29 |
+
{"step": 280, "total_steps": 28610, "tokens": 587202560, "source": "fineweb_hq", "phase": 1, "loss": 5.444625377655029, "ce_loss": 5.424403190612793, "logit_z_loss": 6.937403976917267, "router_aux_loss": 1.0929326713085175, "router_z_loss": 8.59907180070877, "routing_entropy_normalized": 0.9221462829721346, "learning_rate": 0.0008399999999999999, "grad_norm": 0.28332892060279846, "tokens_per_second": 3369761.575879382, "eta_seconds": 17631.014783143997, "data_wait_seconds": 2.0831823348999023e-05}
|
| 30 |
+
{"step": 290, "total_steps": 28610, "tokens": 608174080, "source": "fineweb_edu", "phase": 1, "loss": 5.458544015884399, "ce_loss": 5.438980400562286, "logit_z_loss": 7.198760628700256, "router_aux_loss": 1.1046753525733948, "router_z_loss": 7.796859443187714, "routing_entropy_normalized": 0.9172121723952192, "learning_rate": 0.00087, "grad_norm": 0.3885813057422638, "tokens_per_second": 3946078.290696833, "eta_seconds": 15050.726383209229, "data_wait_seconds": 1.8924474716186523e-05}
|
| 31 |
+
{"step": 300, "total_steps": 28610, "tokens": 629145600, "source": "fineweb_hq", "phase": 1, "loss": 5.371817767620087, "ce_loss": 5.352827608585358, "logit_z_loss": 7.224969208240509, "router_aux_loss": 1.1201239973306656, "router_z_loss": 7.066534221172333, "routing_entropy_normalized": 0.9125549541983051, "learning_rate": 0.0009, "grad_norm": 0.39654383063316345, "tokens_per_second": 4044506.3058676585, "eta_seconds": 14679.263334035873, "data_wait_seconds": 2.2023916244506836e-05}
|
| 32 |
+
{"step": 310, "total_steps": 28610, "tokens": 650117120, "source": "fineweb_hq", "phase": 1, "loss": 5.344034552574158, "ce_loss": 5.324778199195862, "logit_z_loss": 7.347847044467926, "router_aux_loss": 1.1336359530687332, "router_z_loss": 7.185166895389557, "routing_entropy_normalized": 0.9100935114274862, "learning_rate": 0.00093, "grad_norm": 0.28645825386047363, "tokens_per_second": 3722775.438646952, "eta_seconds": 15942.245934009552, "data_wait_seconds": 2.3484230041503906e-05}
|
| 33 |
+
{"step": 320, "total_steps": 28610, "tokens": 671088640, "source": "fineweb_hq", "phase": 1, "loss": 5.322923719882965, "ce_loss": 5.304147362709045, "logit_z_loss": 7.559026896953583, "router_aux_loss": 1.1401750296354294, "router_z_loss": 6.618620872497559, "routing_entropy_normalized": 0.9063468496251164, "learning_rate": 0.0009599999999999999, "grad_norm": 0.24293020367622375, "tokens_per_second": 3538588.621823214, "eta_seconds": 16766.128086805344, "data_wait_seconds": 2.002716064453125e-05}
|
| 34 |
+
{"step": 330, "total_steps": 28610, "tokens": 692060160, "source": "fineweb_hq", "phase": 1, "loss": 5.253985226154327, "ce_loss": 5.235036313533783, "logit_z_loss": 7.848530113697052, "router_aux_loss": 1.1589565426111221, "router_z_loss": 6.57447612285614, "routing_entropy_normalized": 0.8965382428093612, "learning_rate": 0.00099, "grad_norm": 0.27227944135665894, "tokens_per_second": 3994233.3172515226, "eta_seconds": 14848.270956993103, "data_wait_seconds": 1.8984079360961914e-05}
|
| 35 |
+
{"step": 340, "total_steps": 28610, "tokens": 713031680, "source": "fineweb_hq", "phase": 1, "loss": 5.239398002624512, "ce_loss": 5.220990836620331, "logit_z_loss": 8.02791178226471, "router_aux_loss": 1.1757470071315765, "router_z_loss": 5.846938669681549, "routing_entropy_normalized": 0.8859969556602151, "learning_rate": 0.00102, "grad_norm": 0.327094703912735, "tokens_per_second": 3421397.879941322, "eta_seconds": 17328.147476673126, "data_wait_seconds": 2.193450927734375e-05}
|
| 36 |
+
{"step": 350, "total_steps": 28610, "tokens": 734003200, "source": "fineweb_edu", "phase": 1, "loss": 5.260049343109131, "ce_loss": 5.241715848445892, "logit_z_loss": 7.85910838842392, "router_aux_loss": 1.1920885890722275, "router_z_loss": 5.626796901226044, "routing_entropy_normalized": 0.8691016782812263, "learning_rate": 0.0010500000000000002, "grad_norm": 0.4705367982387543, "tokens_per_second": 3794660.4059493053, "eta_seconds": 15618.134214878082, "data_wait_seconds": 1.952052116394043e-05}
|
| 37 |
+
{"step": 360, "total_steps": 28610, "tokens": 754974720, "source": "fineweb_hq", "phase": 1, "loss": 5.195451378822327, "ce_loss": 5.17672061920166, "logit_z_loss": 8.42539668083191, "router_aux_loss": 1.223222330212593, "router_z_loss": 5.656034290790558, "routing_entropy_normalized": 0.8833911314073039, "learning_rate": 0.00108, "grad_norm": 0.29895609617233276, "tokens_per_second": 3511669.379188852, "eta_seconds": 16870.763617753983, "data_wait_seconds": 2.7626752853393555e-05}
|
| 38 |
+
{"step": 370, "total_steps": 28610, "tokens": 775946240, "source": "fineweb_edu", "phase": 1, "loss": 5.225911021232605, "ce_loss": 5.207927107810974, "logit_z_loss": 7.770734369754791, "router_aux_loss": 1.2471286058425903, "router_z_loss": 4.735528767108917, "routing_entropy_normalized": 0.8719184432847817, "learning_rate": 0.00111, "grad_norm": 0.5935752391815186, "tokens_per_second": 3798979.7032318194, "eta_seconds": 15589.336376190186, "data_wait_seconds": 1.7762184143066406e-05}
|
| 39 |
+
{"step": 380, "total_steps": 28610, "tokens": 796917760, "source": "fineweb_hq", "phase": 1, "loss": 5.149052083492279, "ce_loss": 5.131161630153656, "logit_z_loss": 8.167403817176819, "router_aux_loss": 1.2667809426784515, "router_z_loss": 4.40583062171936, "routing_entropy_normalized": 0.8729008473567185, "learning_rate": 0.0011400000000000002, "grad_norm": 0.3236619830131531, "tokens_per_second": 4274623.386485481, "eta_seconds": 13849.781748533249, "data_wait_seconds": 1.627206802368164e-05}
|
| 40 |
+
{"step": 390, "total_steps": 28610, "tokens": 817889280, "source": "fineweb_edu", "phase": 1, "loss": 5.128994345664978, "ce_loss": 5.111594498157501, "logit_z_loss": 8.529348969459534, "router_aux_loss": 1.2694161981344223, "router_z_loss": 3.852686494588852, "routing_entropy_normalized": 0.8660489053634466, "learning_rate": 0.00117, "grad_norm": 0.41736388206481934, "tokens_per_second": 3098391.168666903, "eta_seconds": 19100.761078357697, "data_wait_seconds": 1.7881393432617188e-05}
|
| 41 |
+
{"step": 400, "total_steps": 28610, "tokens": 838860800, "source": "fineweb_hq", "phase": 1, "loss": 5.090546727180481, "ce_loss": 5.073071599006653, "logit_z_loss": 8.49722170829773, "router_aux_loss": 1.2949055582284927, "router_z_loss": 3.676444709300995, "routing_entropy_normalized": 0.8672269490131654, "learning_rate": 0.0012, "grad_norm": 0.36304622888565063, "tokens_per_second": 3477695.175258208, "eta_seconds": 17011.455846071243, "data_wait_seconds": 2.0325183868408203e-05}
|
| 42 |
+
{"step": 410, "total_steps": 28610, "tokens": 859832320, "source": "fineweb_hq", "phase": 1, "loss": 5.064834654331207, "ce_loss": 5.047110915184021, "logit_z_loss": 8.8423353433609, "router_aux_loss": 1.3084731101989746, "router_z_loss": 3.7548621892929077, "routing_entropy_normalized": 0.8668069846089523, "learning_rate": 0.00123, "grad_norm": 0.285069078207016, "tokens_per_second": 3221232.3926164685, "eta_seconds": 18359.335556030273, "data_wait_seconds": 0.08201679587364197}
|
| 43 |
+
{"step": 420, "total_steps": 28610, "tokens": 880803840, "source": "fineweb_hq", "phase": 1, "loss": 5.026031970977783, "ce_loss": 5.00834858417511, "logit_z_loss": 9.209359288215637, "router_aux_loss": 1.340689778327942, "router_z_loss": 3.3554183542728424, "routing_entropy_normalized": 0.8624064720336505, "learning_rate": 0.00126, "grad_norm": 0.28937631845474243, "tokens_per_second": 3243420.811924807, "eta_seconds": 18227.272471904755, "data_wait_seconds": 2.0205974578857422e-05}
|
| 44 |
+
{"step": 430, "total_steps": 28610, "tokens": 901775360, "source": "fineweb_hq", "phase": 1, "loss": 4.996566116809845, "ce_loss": 4.978649914264679, "logit_z_loss": 8.815405368804932, "router_aux_loss": 1.3385442197322845, "router_z_loss": 3.649264484643936, "routing_entropy_normalized": 0.8658562643781956, "learning_rate": 0.0012900000000000001, "grad_norm": 0.3979099988937378, "tokens_per_second": 2711752.3787492486, "eta_seconds": 21793.19314813614, "data_wait_seconds": 2.3126602172851562e-05}
|
| 45 |
+
{"step": 440, "total_steps": 28610, "tokens": 922746880, "source": "fineweb_hq", "phase": 1, "loss": 5.012665808200836, "ce_loss": 4.995199501514435, "logit_z_loss": 10.034671306610107, "router_aux_loss": 1.374913051724434, "router_z_loss": 2.7138236314058304, "routing_entropy_normalized": 0.8335043520996626, "learning_rate": 0.00132, "grad_norm": 0.4830191731452942, "tokens_per_second": 3771752.3189891055, "eta_seconds": 15662.951022148132, "data_wait_seconds": 1.800060272216797e-05}
|
| 46 |
+
{"step": 450, "total_steps": 28610, "tokens": 943718400, "source": "fineweb_edu", "phase": 1, "loss": 5.011710703372955, "ce_loss": 4.99358743429184, "logit_z_loss": 8.91873013973236, "router_aux_loss": 1.3978781700134277, "router_z_loss": 3.252637028694153, "routing_entropy_normalized": 0.8358440784948558, "learning_rate": 0.00135, "grad_norm": 0.4271644651889801, "tokens_per_second": 3261740.5741507495, "eta_seconds": 18105.609252929688, "data_wait_seconds": 2.333521842956543e-05}
|
| 47 |
+
{"step": 460, "total_steps": 28610, "tokens": 964689920, "source": "fineweb_hq", "phase": 1, "loss": 4.97152978181839, "ce_loss": 4.9525768756866455, "logit_z_loss": 9.51468300819397, "router_aux_loss": 1.4543810337781906, "router_z_loss": 3.457518130540848, "routing_entropy_normalized": 0.8401015661263701, "learning_rate": 0.0013800000000000002, "grad_norm": 0.4248215854167938, "tokens_per_second": 3271589.4996804693, "eta_seconds": 18044.693200588226, "data_wait_seconds": 2.390146255493164e-05}
|
| 48 |
+
{"step": 470, "total_steps": 28610, "tokens": 985661440, "source": "fineweb_edu", "phase": 1, "loss": 4.94115424156189, "ce_loss": 4.9231467843055725, "logit_z_loss": 9.461663722991943, "router_aux_loss": 1.4546733051538467, "router_z_loss": 2.514692634344101, "routing_entropy_normalized": 0.8327621696468369, "learning_rate": 0.00141, "grad_norm": 0.3548778295516968, "tokens_per_second": 4198693.589163896, "eta_seconds": 14055.2903008461, "data_wait_seconds": 2.4110078811645508e-05}
|
| 49 |
+
{"step": 480, "total_steps": 28610, "tokens": 1006632960, "source": "fineweb_hq", "phase": 1, "loss": 4.902189314365387, "ce_loss": 4.883037090301514, "logit_z_loss": 9.651476621627808, "router_aux_loss": 1.4716897010803223, "router_z_loss": 3.470106780529022, "routing_entropy_normalized": 0.8425183360603846, "learning_rate": 0.0014399999999999999, "grad_norm": 0.3314315974712372, "tokens_per_second": 3967594.0942508336, "eta_seconds": 14868.679698228836, "data_wait_seconds": 2.6285648345947266e-05}
|
| 50 |
+
{"step": 490, "total_steps": 28610, "tokens": 1027604480, "source": "fineweb_edu", "phase": 1, "loss": 4.898950099945068, "ce_loss": 4.880696892738342, "logit_z_loss": 10.149771094322205, "router_aux_loss": 1.457972526550293, "router_z_loss": 2.6584786474704742, "routing_entropy_normalized": 0.8454257597435149, "learning_rate": 0.00147, "grad_norm": 0.4022069275379181, "tokens_per_second": 3920839.035617522, "eta_seconds": 15040.636380195618, "data_wait_seconds": 1.913309097290039e-05}
|
| 51 |
+
{"step": 500, "total_steps": 28610, "tokens": 1048576000, "source": "fineweb_hq", "phase": 1, "loss": 4.847365617752075, "ce_loss": 4.828476548194885, "logit_z_loss": 9.975858449935913, "router_aux_loss": 1.48147913813591, "router_z_loss": 3.0766870975494385, "routing_entropy_normalized": 0.84149642962929, "learning_rate": 0.0015, "grad_norm": 0.36526528000831604, "tokens_per_second": 3910478.4742608904, "eta_seconds": 15075.122675657272, "data_wait_seconds": 2.5570392608642578e-05}
|
| 52 |
+
{"step": 510, "total_steps": 28610, "tokens": 1069547520, "source": "fineweb_hq", "phase": 1, "loss": 4.814844310283661, "ce_loss": 4.796235978603363, "logit_z_loss": 10.289829730987549, "router_aux_loss": 1.5016645789146423, "router_z_loss": 2.562759593129158, "routing_entropy_normalized": 0.8309261884498913, "learning_rate": 0.0015300000000000001, "grad_norm": 0.3031240701675415, "tokens_per_second": 3285293.2215354447, "eta_seconds": 17937.507317066193, "data_wait_seconds": 1.9997358322143555e-05}
|
| 53 |
+
{"step": 520, "total_steps": 28610, "tokens": 1090519040, "source": "fineweb_hq", "phase": 1, "loss": 4.787340223789215, "ce_loss": 4.76912522315979, "logit_z_loss": 10.826215505599976, "router_aux_loss": 1.4969516545534134, "router_z_loss": 2.1627984941005707, "routing_entropy_normalized": 0.8335338224968717, "learning_rate": 0.00156, "grad_norm": 0.3507811427116394, "tokens_per_second": 3777777.558257838, "eta_seconds": 15593.559644937515, "data_wait_seconds": 2.1338462829589844e-05}
|
| 54 |
+
{"step": 530, "total_steps": 28610, "tokens": 1111490560, "source": "fineweb_hq", "phase": 1, "loss": 4.750087261199951, "ce_loss": 4.731242060661316, "logit_z_loss": 10.154732346534729, "router_aux_loss": 1.5312494933605194, "router_z_loss": 2.5173303484916687, "routing_entropy_normalized": 0.8234943176656766, "learning_rate": 0.00159, "grad_norm": 0.3950449228286743, "tokens_per_second": 3994685.173093473, "eta_seconds": 14741.594295501709, "data_wait_seconds": 2.0802021026611328e-05}
|
| 55 |
+
{"step": 540, "total_steps": 28610, "tokens": 1132462080, "source": "fineweb_hq", "phase": 1, "loss": 4.773451924324036, "ce_loss": 4.755377769470215, "logit_z_loss": 10.728098392486572, "router_aux_loss": 1.5105438083410263, "router_z_loss": 1.895902305841446, "routing_entropy_normalized": 0.8281748098738861, "learning_rate": 0.0016200000000000001, "grad_norm": 0.2856164574623108, "tokens_per_second": 3237227.897222266, "eta_seconds": 18184.402985811234, "data_wait_seconds": 2.2113323211669922e-05}
|
| 56 |
+
{"step": 550, "total_steps": 28610, "tokens": 1153433600, "source": "fineweb_edu", "phase": 1, "loss": 4.7697367668151855, "ce_loss": 4.751023888587952, "logit_z_loss": 10.269596457481384, "router_aux_loss": 1.522117242217064, "router_z_loss": 2.464825749397278, "routing_entropy_normalized": 0.8268924730287408, "learning_rate": 0.0016500000000000002, "grad_norm": 0.3417162001132965, "tokens_per_second": 4113319.2611890826, "eta_seconds": 14306.228469848633, "data_wait_seconds": 2.1398067474365234e-05}
|
| 57 |
+
{"step": 560, "total_steps": 28610, "tokens": 1174405120, "source": "fineweb_hq", "phase": 1, "loss": 4.73752373456955, "ce_loss": 4.71814239025116, "logit_z_loss": 10.787253499031067, "router_aux_loss": 1.5451249331235886, "router_z_loss": 2.8512998819351196, "routing_entropy_normalized": 0.8299761225786543, "learning_rate": 0.0016799999999999999, "grad_norm": 0.2863750457763672, "tokens_per_second": 4107901.715805139, "eta_seconds": 14319.990513324738, "data_wait_seconds": 1.773238182067871e-05}
|
| 58 |
+
{"step": 570, "total_steps": 28610, "tokens": 1195376640, "source": "fineweb_edu", "phase": 1, "loss": 4.693858504295349, "ce_loss": 4.675326585769653, "logit_z_loss": 10.672502160072327, "router_aux_loss": 1.5508647412061691, "router_z_loss": 1.9561002552509308, "routing_entropy_normalized": 0.8406210676870906, "learning_rate": 0.00171, "grad_norm": 0.2887071669101715, "tokens_per_second": 4110416.6146404147, "eta_seconds": 14306.126992225647, "data_wait_seconds": 2.187490463256836e-05}
|
| 59 |
+
{"step": 580, "total_steps": 28610, "tokens": 1216348160, "source": "fineweb_hq", "phase": 1, "loss": 4.668807566165924, "ce_loss": 4.650371611118317, "logit_z_loss": 10.875535607337952, "router_aux_loss": 1.5739277303218842, "router_z_loss": 1.6090629547834396, "routing_entropy_normalized": 0.825997802728899, "learning_rate": 0.00174, "grad_norm": 0.40386706590652466, "tokens_per_second": 4004732.1548217814, "eta_seconds": 14678.427492141724, "data_wait_seconds": 1.9997358322143555e-05}
|
| 60 |
+
{"step": 590, "total_steps": 28610, "tokens": 1237319680, "source": "fineweb_edu", "phase": 1, "loss": 4.661233842372894, "ce_loss": 4.6426496505737305, "logit_z_loss": 10.53859806060791, "router_aux_loss": 1.5447949022054672, "router_z_loss": 2.0824166238307953, "routing_entropy_normalized": 0.8147948104423992, "learning_rate": 0.00177, "grad_norm": 0.37953150272369385, "tokens_per_second": 3354839.0208077533, "eta_seconds": 17515.65385866165, "data_wait_seconds": 1.6123056411743164e-05}
|
| 61 |
+
{"step": 600, "total_steps": 28610, "tokens": 1258291200, "source": "fineweb_hq", "phase": 1, "loss": 4.6461856961250305, "ce_loss": 4.627599895000458, "logit_z_loss": 10.74972689151764, "router_aux_loss": 1.6010109633207321, "router_z_loss": 1.5006704330444336, "routing_entropy_normalized": 0.8078374543586948, "learning_rate": 0.0018, "grad_norm": 0.4571743309497833, "tokens_per_second": 4312127.1617988, "eta_seconds": 13622.331929445269, "data_wait_seconds": 2.086162567138672e-05}
|
| 62 |
+
{"step": 610, "total_steps": 28610, "tokens": 1279262720, "source": "fineweb_hq", "phase": 1, "loss": 4.602349638938904, "ce_loss": 4.583353698253632, "logit_z_loss": 11.201192855834961, "router_aux_loss": 1.5656961351633072, "router_z_loss": 2.2190346717834473, "routing_entropy_normalized": 0.8232387035378077, "learning_rate": 0.00183, "grad_norm": 0.26169553399086, "tokens_per_second": 3849583.796473134, "eta_seconds": 15253.663539886475, "data_wait_seconds": 1.800060272216797e-05}
|
| 63 |
+
{"step": 620, "total_steps": 28610, "tokens": 1300234240, "source": "fineweb_hq", "phase": 1, "loss": 4.602822721004486, "ce_loss": 4.584046304225922, "logit_z_loss": 11.272533416748047, "router_aux_loss": 1.6290462166070938, "router_z_loss": 1.3587355017662048, "routing_entropy_normalized": 0.8025154377816331, "learning_rate": 0.00186, "grad_norm": 0.3588958978652954, "tokens_per_second": 4190568.7294397634, "eta_seconds": 14007.474467039108, "data_wait_seconds": 2.6851892471313477e-05}
|
| 64 |
+
{"step": 630, "total_steps": 28610, "tokens": 1321205760, "source": "fineweb_hq", "phase": 1, "loss": 4.582616686820984, "ce_loss": 4.562750041484833, "logit_z_loss": 11.323569059371948, "router_aux_loss": 1.6738543808460236, "router_z_loss": 1.9957774057984352, "routing_entropy_normalized": 0.7996818237684693, "learning_rate": 0.0018900000000000002, "grad_norm": 0.3162058889865875, "tokens_per_second": 3398635.576695529, "eta_seconds": 17265.255905151367, "data_wait_seconds": 2.3037195205688477e-05}
|
| 65 |
+
{"step": 640, "total_steps": 28610, "tokens": 1342177280, "source": "fineweb_hq", "phase": 1, "loss": 4.556235313415527, "ce_loss": 4.537401795387268, "logit_z_loss": 11.744758009910583, "router_aux_loss": 1.6015048623085022, "router_z_loss": 1.6439045071601868, "routing_entropy_normalized": 0.8044678263578776, "learning_rate": 0.0019199999999999998, "grad_norm": 0.35044100880622864, "tokens_per_second": 3756238.5086336415, "eta_seconds": 15615.978938817978, "data_wait_seconds": 2.0772218704223633e-05}
|
| 66 |
+
{"step": 650, "total_steps": 28610, "tokens": 1363148800, "source": "fineweb_edu", "phase": 1, "loss": 4.518477559089661, "ce_loss": 4.499395430088043, "logit_z_loss": 11.374523282051086, "router_aux_loss": 1.6293142884969711, "router_z_loss": 1.6516766399145126, "routing_entropy_normalized": 0.7947154529089143, "learning_rate": 0.00195, "grad_norm": 0.38360729813575745, "tokens_per_second": 3316277.8402965195, "eta_seconds": 17681.380373954773, "data_wait_seconds": 1.9222497940063477e-05}
|
| 67 |
+
{"step": 660, "total_steps": 28610, "tokens": 1384120320, "source": "fineweb_hq", "phase": 1, "loss": 4.4425437450408936, "ce_loss": 4.423097372055054, "logit_z_loss": 11.37237823009491, "router_aux_loss": 1.6626565903425217, "router_z_loss": 1.6825388222932816, "routing_entropy_normalized": 0.8031857765339157, "learning_rate": 0.00198, "grad_norm": 0.3046814501285553, "tokens_per_second": 3643489.94522065, "eta_seconds": 16087.70691871643, "data_wait_seconds": 1.7255544662475586e-05}
|
| 68 |
+
{"step": 670, "total_steps": 28610, "tokens": 1405091840, "source": "fineweb_edu", "phase": 1, "loss": 4.475962996482849, "ce_loss": 4.456783711910248, "logit_z_loss": 11.148050785064697, "router_aux_loss": 1.645657703280449, "router_z_loss": 1.6077712774276733, "routing_entropy_normalized": 0.7975695938988208, "learning_rate": 0.00201, "grad_norm": 0.33695414662361145, "tokens_per_second": 3845425.758889983, "eta_seconds": 15237.4354763031, "data_wait_seconds": 2.0891427993774414e-05}
|
| 69 |
+
{"step": 680, "total_steps": 28610, "tokens": 1426063360, "source": "fineweb_hq", "phase": 1, "loss": 4.378860116004944, "ce_loss": 4.360008001327515, "logit_z_loss": 11.952872514724731, "router_aux_loss": 1.6375892758369446, "router_z_loss": 1.280927300453186, "routing_entropy_normalized": 0.7981218237596236, "learning_rate": 0.00204, "grad_norm": 0.3253091871738434, "tokens_per_second": 3811494.812161504, "eta_seconds": 15367.581027030945, "data_wait_seconds": 2.1219253540039062e-05}
|
| 70 |
+
{"step": 690, "total_steps": 28610, "tokens": 1447034880, "source": "fineweb_edu", "phase": 1, "loss": 4.3715115785598755, "ce_loss": 4.352568805217743, "logit_z_loss": 11.637056589126587, "router_aux_loss": 1.6241776943206787, "router_z_loss": 1.5373363643884659, "routing_entropy_normalized": 0.8207539881979657, "learning_rate": 0.00207, "grad_norm": 0.35822823643684387, "tokens_per_second": 3583570.173558502, "eta_seconds": 16339.148113250732, "data_wait_seconds": 2.1636486053466797e-05}
|
| 71 |
+
{"step": 700, "total_steps": 28610, "tokens": 1468006400, "source": "fineweb_hq", "phase": 1, "loss": 4.312666177749634, "ce_loss": 4.293725490570068, "logit_z_loss": 12.11507797241211, "router_aux_loss": 1.6435659229755402, "router_z_loss": 1.2935548275709152, "routing_entropy_normalized": 0.8119192774798145, "learning_rate": 0.0021000000000000003, "grad_norm": 0.300942599773407, "tokens_per_second": 3136614.7157123517, "eta_seconds": 18660.72744822502, "data_wait_seconds": 0.4307662844657898}
|
| 72 |
+
{"step": 710, "total_steps": 28610, "tokens": 1488977920, "source": "fineweb_hq", "phase": 1, "loss": 4.183160781860352, "ce_loss": 4.163766860961914, "logit_z_loss": 12.157939314842224, "router_aux_loss": 1.622790277004242, "router_z_loss": 1.9503589868545532, "routing_entropy_normalized": 0.8265703172821873, "learning_rate": 0.00213, "grad_norm": 0.2843954861164093, "tokens_per_second": 4206488.292379407, "eta_seconds": 13909.593164920807, "data_wait_seconds": 2.3156404495239258e-05}
|
| 73 |
+
{"step": 720, "total_steps": 28610, "tokens": 1509949440, "source": "fineweb_hq", "phase": 1, "loss": 4.1320576667785645, "ce_loss": 4.112650454044342, "logit_z_loss": 11.913991093635559, "router_aux_loss": 1.6702817976474762, "router_z_loss": 1.5128950029611588, "routing_entropy_normalized": 0.7667686280578151, "learning_rate": 0.00216, "grad_norm": 0.3295558989048004, "tokens_per_second": 3582569.064308919, "eta_seconds": 16326.152610063553, "data_wait_seconds": 2.2679567337036133e-05}
|
| 74 |
+
{"step": 730, "total_steps": 28610, "tokens": 1530920960, "source": "fineweb_hq", "phase": 1, "loss": 4.026218414306641, "ce_loss": 4.007347136735916, "logit_z_loss": 12.179679989814758, "router_aux_loss": 1.6417283415794373, "router_z_loss": 1.235919177532196, "routing_entropy_normalized": 0.7842817565096617, "learning_rate": 0.00219, "grad_norm": 0.28047820925712585, "tokens_per_second": 3723436.517013419, "eta_seconds": 15702.858768463135, "data_wait_seconds": 2.4765729904174805e-05}
|
| 75 |
+
{"step": 740, "total_steps": 28610, "tokens": 1551892480, "source": "fineweb_hq", "phase": 1, "loss": 3.989847034215927, "ce_loss": 3.9702339470386505, "logit_z_loss": 12.447178483009338, "router_aux_loss": 1.6604880839586258, "router_z_loss": 1.7635953277349472, "routing_entropy_normalized": 0.7562691363023605, "learning_rate": 0.00222, "grad_norm": 0.346403032541275, "tokens_per_second": 3894074.253051765, "eta_seconds": 15009.376411914825, "data_wait_seconds": 2.014636993408203e-05}
|
| 76 |
+
{"step": 750, "total_steps": 28610, "tokens": 1572864000, "source": "fineweb_edu", "phase": 1, "loss": 4.258761405944824, "ce_loss": 4.239522159099579, "logit_z_loss": 12.369433999061584, "router_aux_loss": 1.6498509645462036, "router_z_loss": 1.5039474442601204, "routing_entropy_normalized": 0.7945254378483197, "learning_rate": 0.00225, "grad_norm": 0.3874565660953522, "tokens_per_second": 3626962.30448711, "eta_seconds": 16108.977655410767, "data_wait_seconds": 1.9729137420654297e-05}
|
| 77 |
+
{"step": 760, "total_steps": 28610, "tokens": 1593835520, "source": "fineweb_hq", "phase": 1, "loss": 3.9048891365528107, "ce_loss": 3.885349154472351, "logit_z_loss": 12.790281653404236, "router_aux_loss": 1.701425939798355, "router_z_loss": 1.2466687560081482, "routing_entropy_normalized": 0.7865120201424554, "learning_rate": 0.0022800000000000003, "grad_norm": 0.29702228307724, "tokens_per_second": 4180731.0651545026, "eta_seconds": 13970.208150148392, "data_wait_seconds": 2.4020671844482422e-05}
|
| 78 |
+
{"step": 770, "total_steps": 28610, "tokens": 1614807040, "source": "fineweb_edu", "phase": 1, "loss": 4.154159486293793, "ce_loss": 4.134239077568054, "logit_z_loss": 12.853577375411987, "router_aux_loss": 1.7273095399141312, "router_z_loss": 1.3619235306978226, "routing_entropy_normalized": 0.7853914621656923, "learning_rate": 0.00231, "grad_norm": 0.3584041893482208, "tokens_per_second": 3708947.911804715, "eta_seconds": 15741.58307647705, "data_wait_seconds": 2.0563602447509766e-05}
|
| 79 |
+
{"step": 780, "total_steps": 28610, "tokens": 1635778560, "source": "fineweb_hq", "phase": 1, "loss": 3.835381865501404, "ce_loss": 3.8157514929771423, "logit_z_loss": 12.789737343788147, "router_aux_loss": 1.705956608057022, "router_z_loss": 1.291787475347519, "routing_entropy_normalized": 0.7713168525771344, "learning_rate": 0.00234, "grad_norm": 0.40305009484291077, "tokens_per_second": 3415538.7750444957, "eta_seconds": 17087.71119403839, "data_wait_seconds": 2.6911497116088867e-05}
|
| 80 |
+
{"step": 790, "total_steps": 28610, "tokens": 1656750080, "source": "fineweb_edu", "phase": 1, "loss": 4.152964353561401, "ce_loss": 4.133545517921448, "logit_z_loss": 12.332417488098145, "router_aux_loss": 1.7060890197753906, "router_z_loss": 1.1245883628726006, "routing_entropy_normalized": 0.7657672862067394, "learning_rate": 0.00237, "grad_norm": 0.46772322058677673, "tokens_per_second": 3153174.1463991515, "eta_seconds": 18502.86915063858, "data_wait_seconds": 2.1070241928100586e-05}
|
| 81 |
+
{"step": 800, "total_steps": 28610, "tokens": 1677721600, "source": "fineweb_hq", "phase": 1, "loss": 3.761558085680008, "ce_loss": 3.741902142763138, "logit_z_loss": 12.860188007354736, "router_aux_loss": 1.7149255573749542, "router_z_loss": 1.2206990122795105, "routing_entropy_normalized": 0.7890314993665327, "learning_rate": 0.0024, "grad_norm": 0.32772231101989746, "tokens_per_second": 4486069.245902081, "eta_seconds": 13000.645759820938, "data_wait_seconds": 2.047419548034668e-05}
|
| 82 |
+
{"step": 810, "total_steps": 28610, "tokens": 1698693120, "source": "fineweb_hq", "phase": 1, "loss": 3.7172193825244904, "ce_loss": 3.6978485584259033, "logit_z_loss": 13.18132734298706, "router_aux_loss": 1.6983977407217026, "router_z_loss": 1.0686743780970573, "routing_entropy_normalized": 0.7752630754364466, "learning_rate": 0.0024300000000000003, "grad_norm": 0.27187544107437134, "tokens_per_second": 3851637.2644714345, "eta_seconds": 15136.634526252747, "data_wait_seconds": 2.2709369659423828e-05}
|
| 83 |
+
{"step": 820, "total_steps": 28610, "tokens": 1719664640, "source": "fineweb_hq", "phase": 1, "loss": 3.7155794203281403, "ce_loss": 3.696614146232605, "logit_z_loss": 12.875759959220886, "router_aux_loss": 1.6699260771274567, "router_z_loss": 0.9784659892320633, "routing_entropy_normalized": 0.7752238752221113, "learning_rate": 0.00246, "grad_norm": 0.3380814790725708, "tokens_per_second": 4257191.374413602, "eta_seconds": 13689.74259185791, "data_wait_seconds": 2.1845102310180664e-05}
|
| 84 |
+
{"step": 830, "total_steps": 28610, "tokens": 1740636160, "source": "fineweb_hq", "phase": 1, "loss": 3.7125514447689056, "ce_loss": 3.693806678056717, "logit_z_loss": 13.074058175086975, "router_aux_loss": 1.6327433735132217, "router_z_loss": 1.1098729148507118, "routing_entropy_normalized": 0.7661689784290413, "learning_rate": 0.00249, "grad_norm": 0.4026620388031006, "tokens_per_second": 3827754.197857899, "eta_seconds": 15220.121133327484, "data_wait_seconds": 2.5928020477294922e-05}
|
| 85 |
+
{"step": 840, "total_steps": 28610, "tokens": 1761607680, "source": "fineweb_hq", "phase": 1, "loss": 3.7102659046649933, "ce_loss": 3.6910923421382904, "logit_z_loss": 13.138167023658752, "router_aux_loss": 1.6986212879419327, "router_z_loss": 0.8735387995839119, "routing_entropy_normalized": 0.7574490153708061, "learning_rate": 0.00252, "grad_norm": 0.30801060795783997, "tokens_per_second": 4198816.048648469, "eta_seconds": 13870.079175949097, "data_wait_seconds": 2.1010637283325195e-05}
|
| 86 |
+
{"step": 850, "total_steps": 28610, "tokens": 1782579200, "source": "fineweb_edu", "phase": 1, "loss": 4.09829968214035, "ce_loss": 4.078723132610321, "logit_z_loss": 13.299319982528687, "router_aux_loss": 1.706539511680603, "router_z_loss": 1.1810516864061356, "routing_entropy_normalized": 0.794611320065966, "learning_rate": 0.00255, "grad_norm": 0.4702201783657074, "tokens_per_second": 3717805.102345282, "eta_seconds": 15658.954118728638, "data_wait_seconds": 2.181529998779297e-05}
|
| 87 |
+
{"step": 860, "total_steps": 28610, "tokens": 1803550720, "source": "fineweb_hq", "phase": 1, "loss": 3.7052374482154846, "ce_loss": 3.684843987226486, "logit_z_loss": 12.924830555915833, "router_aux_loss": 1.7955583035945892, "router_z_loss": 1.1454517841339111, "routing_entropy_normalized": 0.7589951169785014, "learning_rate": 0.0025800000000000003, "grad_norm": 0.33521220088005066, "tokens_per_second": 3558473.307386744, "eta_seconds": 16354.195457696915, "data_wait_seconds": 2.9385089874267578e-05}
|
| 88 |
+
{"step": 870, "total_steps": 28610, "tokens": 1824522240, "source": "fineweb_edu", "phase": 1, "loss": 3.977514296770096, "ce_loss": 3.955770641565323, "logit_z_loss": 13.186537623405457, "router_aux_loss": 1.7855572402477264, "router_z_loss": 2.569445922970772, "routing_entropy_normalized": 0.7666084066508106, "learning_rate": 0.00261, "grad_norm": 0.3989207446575165, "tokens_per_second": 3524651.16889396, "eta_seconds": 16505.178439617157, "data_wait_seconds": 2.008676528930664e-05}
|
| 89 |
+
{"step": 880, "total_steps": 28610, "tokens": 1845493760, "source": "fineweb_hq", "phase": 1, "loss": 3.6696519553661346, "ce_loss": 3.6492239832878113, "logit_z_loss": 13.389381766319275, "router_aux_loss": 1.7835367769002914, "router_z_loss": 1.2537417337298393, "routing_entropy_normalized": 0.7679324976278832, "learning_rate": 0.00264, "grad_norm": 0.3279078006744385, "tokens_per_second": 4268992.312113424, "eta_seconds": 13622.424382209778, "data_wait_seconds": 1.8805265426635742e-05}
|
| 90 |
+
{"step": 890, "total_steps": 28610, "tokens": 1866465280, "source": "fineweb_edu", "phase": 1, "loss": 3.996062844991684, "ce_loss": 3.9759906232357025, "logit_z_loss": 13.08531665802002, "router_aux_loss": 1.7740143686532974, "router_z_loss": 1.0235832035541534, "routing_entropy_normalized": 0.7755338138938909, "learning_rate": 0.00267, "grad_norm": 0.30120849609375, "tokens_per_second": 3833649.2002301733, "eta_seconds": 15163.894869804382, "data_wait_seconds": 2.1219253540039062e-05}
|
| 91 |
+
{"step": 900, "total_steps": 28610, "tokens": 1887436800, "source": "fineweb_hq", "phase": 1, "loss": 3.646775335073471, "ce_loss": 3.6263699531555176, "logit_z_loss": 13.379936933517456, "router_aux_loss": 1.7992166578769684, "router_z_loss": 1.0752902254462242, "routing_entropy_normalized": 0.7632551451260177, "learning_rate": 0.0027, "grad_norm": 0.29987722635269165, "tokens_per_second": 4073799.3752869717, "eta_seconds": 14264.836474895477, "data_wait_seconds": 1.9311904907226562e-05}
|
| 92 |
+
{"step": 910, "total_steps": 28610, "tokens": 1908408320, "source": "fineweb_hq", "phase": 1, "loss": 3.6034410297870636, "ce_loss": 3.583277016878128, "logit_z_loss": 13.17744266986847, "router_aux_loss": 1.752038612961769, "router_z_loss": 1.3258824571967125, "routing_entropy_normalized": 0.7488830156609491, "learning_rate": 0.00273, "grad_norm": 0.3001411259174347, "tokens_per_second": 4210439.636479579, "eta_seconds": 13796.922748088838, "data_wait_seconds": 2.22623348236084e-05}
|
| 93 |
+
{"step": 920, "total_steps": 28610, "tokens": 1929379840, "source": "fineweb_hq", "phase": 1, "loss": 3.5841883718967438, "ce_loss": 3.5642730593681335, "logit_z_loss": 13.444478392601013, "router_aux_loss": 1.7360568344593048, "router_z_loss": 1.2103381156921387, "routing_entropy_normalized": 0.7559414349105724, "learning_rate": 0.0027600000000000003, "grad_norm": 0.34887921810150146, "tokens_per_second": 3590792.686947137, "eta_seconds": 16171.955315351486, "data_wait_seconds": 1.8358230590820312e-05}
|
| 94 |
+
{"step": 930, "total_steps": 28610, "tokens": 1950351360, "source": "fineweb_hq", "phase": 1, "loss": 3.5708978474140167, "ce_loss": 3.5510771572589874, "logit_z_loss": 13.872437477111816, "router_aux_loss": 1.7491921335458755, "router_z_loss": 0.9415555894374847, "routing_entropy_normalized": 0.7723916339273227, "learning_rate": 0.00279, "grad_norm": 0.304644376039505, "tokens_per_second": 3686253.455075078, "eta_seconds": 15747.47045135498, "data_wait_seconds": 2.276897430419922e-05}
|
| 95 |
+
{"step": 940, "total_steps": 28610, "tokens": 1971322880, "source": "fineweb_hq", "phase": 1, "loss": 3.600455731153488, "ce_loss": 3.580485314130783, "logit_z_loss": 13.248793244361877, "router_aux_loss": 1.7551615685224533, "router_z_loss": 1.0939159542322159, "routing_entropy_normalized": 0.7618970862816926, "learning_rate": 0.00282, "grad_norm": 0.33136674761772156, "tokens_per_second": 3656848.4852272323, "eta_seconds": 15868.362081289291, "data_wait_seconds": 2.0235776901245117e-05}
|
| 96 |
+
{"step": 950, "total_steps": 28610, "tokens": 1992294400, "source": "fineweb_edu", "phase": 1, "loss": 3.9880300164222717, "ce_loss": 3.968054324388504, "logit_z_loss": 13.120636105537415, "router_aux_loss": 1.7475409358739853, "router_z_loss": 1.188227191567421, "routing_entropy_normalized": 0.7437454133702321, "learning_rate": 0.00285, "grad_norm": 0.32099828124046326, "tokens_per_second": 3152982.001634185, "eta_seconds": 18397.575466632843, "data_wait_seconds": 1.850724220275879e-05}
|
| 97 |
+
{"step": 960, "total_steps": 28610, "tokens": 2013265920, "source": "fineweb_hq", "phase": 1, "loss": 3.579824149608612, "ce_loss": 3.559646785259247, "logit_z_loss": 13.07980501651764, "router_aux_loss": 1.7664236277341843, "router_z_loss": 1.2051368951797485, "routing_entropy_normalized": 0.7310786296562841, "learning_rate": 0.0028799999999999997, "grad_norm": 0.35547634959220886, "tokens_per_second": 3797136.7155960863, "eta_seconds": 15271.04688167572, "data_wait_seconds": 2.0682811737060547e-05}
|
| 98 |
+
{"step": 970, "total_steps": 28610, "tokens": 2034237440, "source": "fineweb_edu", "phase": 1, "loss": 3.983655869960785, "ce_loss": 3.963058203458786, "logit_z_loss": 13.092215418815613, "router_aux_loss": 1.8066293448209763, "router_z_loss": 1.2221563532948494, "routing_entropy_normalized": 0.7500693587069646, "learning_rate": 0.0029100000000000003, "grad_norm": 0.36281490325927734, "tokens_per_second": 3484643.5538551393, "eta_seconds": 16634.493710517883, "data_wait_seconds": 1.96993350982666e-05}
|
| 99 |
+
{"step": 980, "total_steps": 28610, "tokens": 2055208960, "source": "fineweb_hq", "phase": 1, "loss": 3.5286049246788025, "ce_loss": 3.5081448554992676, "logit_z_loss": 13.278219103813171, "router_aux_loss": 1.816750630736351, "router_z_loss": 0.9648462235927582, "routing_entropy_normalized": 0.7327255736342961, "learning_rate": 0.00294, "grad_norm": 0.2528068423271179, "tokens_per_second": 3734904.9811785454, "eta_seconds": 15514.266106367111, "data_wait_seconds": 0.09053659439086914}
|
| 100 |
+
{"step": 990, "total_steps": 28610, "tokens": 2076180480, "source": "fineweb_edu", "phase": 1, "loss": 3.9431776106357574, "ce_loss": 3.923240840435028, "logit_z_loss": 13.035893082618713, "router_aux_loss": 1.7431392669677734, "router_z_loss": 1.2018379122018814, "routing_entropy_normalized": 0.7352828859832841, "learning_rate": 0.00297, "grad_norm": 0.3029248118400574, "tokens_per_second": 4266480.414587309, "eta_seconds": 13576.37504720688, "data_wait_seconds": 2.0176172256469727e-05}
|
| 101 |
+
{"step": 1000, "total_steps": 28610, "tokens": 2097152000, "source": "fineweb_hq", "phase": 1, "loss": 3.498715817928314, "ce_loss": 3.4785490930080414, "logit_z_loss": 13.087662696838379, "router_aux_loss": 1.7868202030658722, "router_z_loss": 0.9897365272045135, "routing_entropy_normalized": 0.7360057230653183, "learning_rate": 0.003, "grad_norm": 0.29909613728523254, "tokens_per_second": 3728867.9674247233, "eta_seconds": 15528.135408878326, "data_wait_seconds": 1.8298625946044922e-05}
|
| 102 |
+
{"step": 1010, "total_steps": 28610, "tokens": 2118123520, "source": "fineweb_hq", "phase": 1, "loss": 3.5159562528133392, "ce_loss": 3.495896190404892, "logit_z_loss": 13.587857961654663, "router_aux_loss": 1.7780744135379791, "router_z_loss": 0.9205065071582794, "routing_entropy_normalized": 0.7503177885476127, "learning_rate": 0.003, "grad_norm": 0.26984965801239014, "tokens_per_second": 3872858.2704762225, "eta_seconds": 14945.394630432129, "data_wait_seconds": 1.9490718841552734e-05}
|
| 103 |
+
{"step": 1020, "total_steps": 28610, "tokens": 2139095040, "source": "fineweb_hq", "phase": 1, "loss": 3.5288613736629486, "ce_loss": 3.508589416742325, "logit_z_loss": 13.038822412490845, "router_aux_loss": 1.795256569981575, "router_z_loss": 1.0154938399791718, "routing_entropy_normalized": 0.7252003158749227, "learning_rate": 0.003, "grad_norm": 0.3407430648803711, "tokens_per_second": 2960840.179740891, "eta_seconds": 19541.89357328415, "data_wait_seconds": 0.4497779607772827}
|
| 104 |
+
{"step": 1030, "total_steps": 28610, "tokens": 2160066560, "source": "fineweb_hq", "phase": 1, "loss": 3.626240164041519, "ce_loss": 3.606031209230423, "logit_z_loss": 13.305169343948364, "router_aux_loss": 1.7828250080347061, "router_z_loss": 1.0502191632986069, "routing_entropy_normalized": 0.7262705535443134, "learning_rate": 0.003, "grad_norm": 0.262196809053421, "tokens_per_second": 4214954.372598984, "eta_seconds": 13722.438500404358, "data_wait_seconds": 2.104043960571289e-05}
|
| 105 |
+
{"step": 1040, "total_steps": 28610, "tokens": 2181038080, "source": "fineweb_hq", "phase": 1, "loss": 3.7123197615146637, "ce_loss": 3.6929733753204346, "logit_z_loss": 13.519355535507202, "router_aux_loss": 1.708261102437973, "router_z_loss": 0.9118833243846893, "routing_entropy_normalized": 0.7511877655474616, "learning_rate": 0.003, "grad_norm": 0.27905601263046265, "tokens_per_second": 3762634.7636347334, "eta_seconds": 15366.487653493881, "data_wait_seconds": 2.065300941467285e-05}
|
| 106 |
+
{"step": 1050, "total_steps": 28610, "tokens": 2202009600, "source": "fineweb_edu", "phase": 1, "loss": 3.8573088943958282, "ce_loss": 3.837806612253189, "logit_z_loss": 13.228194832801819, "router_aux_loss": 1.7022795379161835, "router_z_loss": 1.156775839626789, "routing_entropy_normalized": 0.7381396731319702, "learning_rate": 0.003, "grad_norm": 0.23843508958816528, "tokens_per_second": 4272658.925934075, "eta_seconds": 13527.292985916138, "data_wait_seconds": 1.9252300262451172e-05}
|
| 107 |
+
{"step": 1060, "total_steps": 28610, "tokens": 2222981120, "source": "fineweb_hq", "phase": 1, "loss": 3.7506060004234314, "ce_loss": 3.7317832708358765, "logit_z_loss": 13.307474493980408, "router_aux_loss": 1.6551974415779114, "router_z_loss": 0.9399715140461922, "routing_entropy_normalized": 0.7493790836464712, "learning_rate": 0.003, "grad_norm": 0.2862909138202667, "tokens_per_second": 3537260.1018081536, "eta_seconds": 16333.697816133501, "data_wait_seconds": 2.321600914001465e-05}
|
| 108 |
+
{"step": 1070, "total_steps": 28610, "tokens": 2243952640, "source": "fineweb_edu", "phase": 1, "loss": 3.8805067241191864, "ce_loss": 3.862116038799286, "logit_z_loss": 13.276132106781006, "router_aux_loss": 1.6010790765285492, "router_z_loss": 1.0522751286625862, "routing_entropy_normalized": 0.7645256790837567, "learning_rate": 0.003, "grad_norm": 0.3348839282989502, "tokens_per_second": 3734474.1482988326, "eta_seconds": 15465.51503276825, "data_wait_seconds": 2.0951032638549805e-05}
|
| 109 |
+
{"step": 1080, "total_steps": 28610, "tokens": 2264924160, "source": "fineweb_hq", "phase": 1, "loss": 3.8067366182804108, "ce_loss": 3.788293570280075, "logit_z_loss": 12.920913696289062, "router_aux_loss": 1.625083103775978, "router_z_loss": 0.9000958949327469, "routing_entropy_normalized": 0.7309660063137617, "learning_rate": 0.003, "grad_norm": 0.3295542299747467, "tokens_per_second": 3251460.6826405735, "eta_seconds": 17756.510133504868, "data_wait_seconds": 1.9550323486328125e-05}
|
| 110 |
+
{"step": 1090, "total_steps": 28610, "tokens": 2285895680, "source": "fineweb_edu", "phase": 1, "loss": 3.832129031419754, "ce_loss": 3.8134789168834686, "logit_z_loss": 13.14590072631836, "router_aux_loss": 1.641230508685112, "router_z_loss": 0.9232481569051743, "routing_entropy_normalized": 0.7498681874528876, "learning_rate": 0.003, "grad_norm": 0.3055610954761505, "tokens_per_second": 3810904.7922955374, "eta_seconds": 15144.336105346682, "data_wait_seconds": 2.79843807220459e-05}
|
| 111 |
+
{"step": 1100, "total_steps": 28610, "tokens": 2306867200, "source": "fineweb_hq", "phase": 1, "loss": 3.7737776339054108, "ce_loss": 3.754968374967575, "logit_z_loss": 13.320119023323059, "router_aux_loss": 1.659154325723648, "router_z_loss": 0.8857016563415527, "routing_entropy_normalized": 0.7382274051743517, "learning_rate": 0.003, "grad_norm": 0.2607852816581726, "tokens_per_second": 4280932.609609648, "eta_seconds": 13476.65492105484, "data_wait_seconds": 2.7745962142944336e-05}
|
| 112 |
+
{"step": 1110, "total_steps": 28610, "tokens": 2327838720, "source": "fineweb_hq", "phase": 1, "loss": 3.7609165608882904, "ce_loss": 3.74206081032753, "logit_z_loss": 13.571325421333313, "router_aux_loss": 1.6642631590366364, "router_z_loss": 0.8560161367058754, "routing_entropy_normalized": 0.7237807828903218, "learning_rate": 0.003, "grad_norm": 0.3153557777404785, "tokens_per_second": 3570643.443849993, "eta_seconds": 16151.621103286743, "data_wait_seconds": 2.4199485778808594e-05}
|
| 113 |
+
{"step": 1120, "total_steps": 28610, "tokens": 2348810240, "source": "fineweb_hq", "phase": 1, "loss": 3.7744768261909485, "ce_loss": 3.755663573741913, "logit_z_loss": 13.296347856521606, "router_aux_loss": 1.6524961739778519, "router_z_loss": 0.9586733728647232, "routing_entropy_normalized": 0.7247650319640757, "learning_rate": 0.003, "grad_norm": 0.3128862679004669, "tokens_per_second": 3500674.0884237206, "eta_seconds": 16468.459223508835, "data_wait_seconds": 2.1189451217651367e-05}
|
| 114 |
+
{"step": 1130, "total_steps": 28610, "tokens": 2369781760, "source": "fineweb_hq", "phase": 1, "loss": 3.699469178915024, "ce_loss": 3.680673360824585, "logit_z_loss": 13.578590869903564, "router_aux_loss": 1.643067941069603, "router_z_loss": 1.0072607696056366, "routing_entropy_normalized": 0.753703675455834, "learning_rate": 0.003, "grad_norm": 0.22913803160190582, "tokens_per_second": 3757552.359955014, "eta_seconds": 15337.041626930237, "data_wait_seconds": 1.895427703857422e-05}
|
| 115 |
+
{"step": 1140, "total_steps": 28610, "tokens": 2390753280, "source": "fineweb_hq", "phase": 1, "loss": 3.798104554414749, "ce_loss": 3.779301404953003, "logit_z_loss": 13.141302943229675, "router_aux_loss": 1.6656715273857117, "router_z_loss": 0.8323302268981934, "routing_entropy_normalized": 0.7354007526299524, "learning_rate": 0.003, "grad_norm": 0.2660737931728363, "tokens_per_second": 4087355.0079852673, "eta_seconds": 14094.387526273727, "data_wait_seconds": 2.2679567337036133e-05}
|
| 116 |
+
{"step": 1150, "total_steps": 28610, "tokens": 2411724800, "source": "fineweb_edu", "phase": 1, "loss": 3.788751929998398, "ce_loss": 3.7704198360443115, "logit_z_loss": 13.02107846736908, "router_aux_loss": 1.6133394688367844, "router_z_loss": 0.8965314701199532, "routing_entropy_normalized": 0.7422723447465854, "learning_rate": 0.003, "grad_norm": 0.2548324167728424, "tokens_per_second": 3373163.554943924, "eta_seconds": 17072.33965444565, "data_wait_seconds": 2.276897430419922e-05}
|
| 117 |
+
{"step": 1160, "total_steps": 28610, "tokens": 2432696320, "source": "fineweb_hq", "phase": 1, "loss": 3.7746009826660156, "ce_loss": 3.7559984624385834, "logit_z_loss": 13.245291948318481, "router_aux_loss": 1.634489506483078, "router_z_loss": 0.9330129846930504, "routing_entropy_normalized": 0.7614914118459433, "learning_rate": 0.003, "grad_norm": 0.27654552459716797, "tokens_per_second": 3432249.176082783, "eta_seconds": 16772.331916093826, "data_wait_seconds": 2.7894973754882812e-05}
|
| 118 |
+
{"step": 1170, "total_steps": 28610, "tokens": 2453667840, "source": "fineweb_edu", "phase": 1, "loss": 3.8126309514045715, "ce_loss": 3.794321656227112, "logit_z_loss": 12.75848662853241, "router_aux_loss": 1.5911732763051987, "router_z_loss": 1.121643841266632, "routing_entropy_normalized": 0.7567621164987329, "learning_rate": 0.003, "grad_norm": 0.2750265300273895, "tokens_per_second": 4038915.0687995, "eta_seconds": 14247.84871673584, "data_wait_seconds": 2.422928810119629e-05}
|
| 119 |
+
{"step": 1180, "total_steps": 28610, "tokens": 2474639360, "source": "fineweb_hq", "phase": 1, "loss": 3.6882618963718414, "ce_loss": 3.6700197756290436, "logit_z_loss": 13.273204565048218, "router_aux_loss": 1.6002838760614395, "router_z_loss": 0.9119101241230965, "routing_entropy_normalized": 0.7291675344600705, "learning_rate": 0.003, "grad_norm": 0.21844597160816193, "tokens_per_second": 4080201.0902816583, "eta_seconds": 14098.540264844894, "data_wait_seconds": 2.512335777282715e-05}
|
| 120 |
+
{"step": 1190, "total_steps": 28610, "tokens": 2495610880, "source": "fineweb_edu", "phase": 1, "loss": 3.789828360080719, "ce_loss": 3.7716000080108643, "logit_z_loss": 13.224592685699463, "router_aux_loss": 1.6064669042825699, "router_z_loss": 0.84122733771801, "routing_entropy_normalized": 0.7380769266641601, "learning_rate": 0.003, "grad_norm": 0.2946833670139313, "tokens_per_second": 3118568.051447745, "eta_seconds": 18439.20250940323, "data_wait_seconds": 1.96993350982666e-05}
|
| 121 |
+
{"step": 1200, "total_steps": 28610, "tokens": 2516582400, "source": "fineweb_hq", "phase": 1, "loss": 3.702523797750473, "ce_loss": 3.684811234474182, "logit_z_loss": 13.292378902435303, "router_aux_loss": 1.5608905255794525, "router_z_loss": 0.7743784189224243, "routing_entropy_normalized": 0.7559652148516611, "learning_rate": 0.003, "grad_norm": 0.2550068497657776, "tokens_per_second": 3528846.1758712437, "eta_seconds": 16289.442343235016, "data_wait_seconds": 2.0772218704223633e-05}
|
| 122 |
+
{"step": 1210, "total_steps": 28610, "tokens": 2537553920, "source": "fineweb_hq", "phase": 1, "loss": 3.689670890569687, "ce_loss": 3.6719910800457, "logit_z_loss": 13.187568545341492, "router_aux_loss": 1.5623154789209366, "router_z_loss": 0.7378690391778946, "routing_entropy_normalized": 0.7560030415816092, "learning_rate": 0.003, "grad_norm": 0.21758417785167694, "tokens_per_second": 2970520.135819822, "eta_seconds": 19344.075169563293, "data_wait_seconds": 2.35140323638916e-05}
|
| 123 |
+
{"step": 1220, "total_steps": 28610, "tokens": 2558525440, "source": "fineweb_hq", "phase": 1, "loss": 3.710075467824936, "ce_loss": 3.691920965909958, "logit_z_loss": 12.754634499549866, "router_aux_loss": 1.6133923679590225, "router_z_loss": 0.7451487109065056, "routing_entropy_normalized": 0.7189863490575994, "learning_rate": 0.003, "grad_norm": 0.31864070892333984, "tokens_per_second": 3566225.630880171, "eta_seconds": 16106.94308924675, "data_wait_seconds": 2.1696090698242188e-05}
|
| 124 |
+
{"step": 1230, "total_steps": 28610, "tokens": 2579496960, "source": "fineweb_hq", "phase": 1, "loss": 3.6679110527038574, "ce_loss": 3.649574488401413, "logit_z_loss": 12.893252611160278, "router_aux_loss": 1.6213468313217163, "router_z_loss": 0.8337559998035431, "routing_entropy_normalized": 0.7327050150597194, "learning_rate": 0.003, "grad_norm": 0.2422039955854416, "tokens_per_second": 3972809.317470809, "eta_seconds": 14453.25389957428, "data_wait_seconds": 2.136826515197754e-05}
|
| 125 |
+
{"step": 1240, "total_steps": 28610, "tokens": 2600468480, "source": "fineweb_hq", "phase": 1, "loss": 3.7075601518154144, "ce_loss": 3.6893202662467957, "logit_z_loss": 13.275237917900085, "router_aux_loss": 1.6119788140058517, "router_z_loss": 0.7926401719450951, "routing_entropy_normalized": 0.7354555276925282, "learning_rate": 0.003, "grad_norm": 0.292062371969223, "tokens_per_second": 3978261.0064432584, "eta_seconds": 14428.17606663704, "data_wait_seconds": 1.823902130126953e-05}
|
| 126 |
+
{"step": 1250, "total_steps": 28610, "tokens": 2621440000, "source": "fineweb_edu", "phase": 1, "loss": 3.7909673154354095, "ce_loss": 3.7727760672569275, "logit_z_loss": 12.625570058822632, "router_aux_loss": 1.579139232635498, "router_z_loss": 1.1373146176338196, "routing_entropy_normalized": 0.7273836380911421, "learning_rate": 0.003, "grad_norm": 0.2440190315246582, "tokens_per_second": 3614281.744174097, "eta_seconds": 15875.375186920166, "data_wait_seconds": 1.9103288650512695e-05}
|
| 127 |
+
{"step": 1260, "total_steps": 28610, "tokens": 2642411520, "source": "fineweb_hq", "phase": 1, "loss": 3.6652913987636566, "ce_loss": 3.6465583443641663, "logit_z_loss": 12.954219818115234, "router_aux_loss": 1.6404407620429993, "router_z_loss": 1.0331949144601822, "routing_entropy_normalized": 0.7317011982249283, "learning_rate": 0.003, "grad_norm": 0.23806901276111603, "tokens_per_second": 4298708.305674416, "eta_seconds": 13342.87025809288, "data_wait_seconds": 1.9848346710205078e-05}
|
| 128 |
+
{"step": 1270, "total_steps": 28610, "tokens": 2663383040, "source": "fineweb_edu", "phase": 1, "loss": 3.7676600217819214, "ce_loss": 3.7494727969169617, "logit_z_loss": 12.923764109611511, "router_aux_loss": 1.6103628277778625, "router_z_loss": 0.7911927476525307, "routing_entropy_normalized": 0.7305975573147367, "learning_rate": 0.003, "grad_norm": 0.3513587713241577, "tokens_per_second": 3645744.7186719715, "eta_seconds": 15726.86518239975, "data_wait_seconds": 2.1576881408691406e-05}
|
| 129 |
+
{"step": 1280, "total_steps": 28610, "tokens": 2684354560, "source": "fineweb_hq", "phase": 1, "loss": 3.6495718359947205, "ce_loss": 3.6313765347003937, "logit_z_loss": 12.739686965942383, "router_aux_loss": 1.609713077545166, "router_z_loss": 0.8242181017994881, "routing_entropy_normalized": 0.7378334825768961, "learning_rate": 0.003, "grad_norm": 0.2121456265449524, "tokens_per_second": 3907151.9412103337, "eta_seconds": 14669.294929504395, "data_wait_seconds": 2.3484230041503906e-05}
|
| 130 |
+
{"step": 1290, "total_steps": 28610, "tokens": 2705326080, "source": "fineweb_edu", "phase": 1, "loss": 3.7536254823207855, "ce_loss": 3.73561692237854, "logit_z_loss": 12.611430168151855, "router_aux_loss": 1.593292623758316, "router_z_loss": 0.8144921585917473, "routing_entropy_normalized": 0.719207989798834, "learning_rate": 0.003, "grad_norm": 0.2598116099834442, "tokens_per_second": 3546878.2064695973, "eta_seconds": 16153.414158821106, "data_wait_seconds": 2.5570392608642578e-05}
|
| 131 |
+
{"step": 1300, "total_steps": 28610, "tokens": 2726297600, "source": "fineweb_hq", "phase": 1, "loss": 3.6559227406978607, "ce_loss": 3.6380514800548553, "logit_z_loss": 12.79792046546936, "router_aux_loss": 1.580781266093254, "router_z_loss": 0.783683642745018, "routing_entropy_normalized": 0.7178066877357714, "learning_rate": 0.003, "grad_norm": 0.22197207808494568, "tokens_per_second": 4441019.81731526, "eta_seconds": 12896.411967515945, "data_wait_seconds": 1.946091651916504e-05}
|
| 132 |
+
{"step": 1310, "total_steps": 28610, "tokens": 2747269120, "source": "fineweb_hq", "phase": 1, "loss": 3.646061420440674, "ce_loss": 3.628581613302231, "logit_z_loss": 12.883747458457947, "router_aux_loss": 1.5333556532859802, "router_z_loss": 0.8579466566443443, "routing_entropy_normalized": 0.7411227465487313, "learning_rate": 0.003, "grad_norm": 0.19888465106487274, "tokens_per_second": 3926677.7127674557, "eta_seconds": 14580.328152179718, "data_wait_seconds": 1.9222497940063477e-05}
|
| 133 |
+
{"step": 1320, "total_steps": 28610, "tokens": 2768240640, "source": "fineweb_hq", "phase": 1, "loss": 3.6691146194934845, "ce_loss": 3.6513642966747284, "logit_z_loss": 12.92969274520874, "router_aux_loss": 1.5697881579399109, "router_z_loss": 0.7593527622520924, "routing_entropy_normalized": 0.6882685006291569, "learning_rate": 0.003, "grad_norm": 0.33036792278289795, "tokens_per_second": 3761707.7520159306, "eta_seconds": 15214.174479484558, "data_wait_seconds": 2.276897430419922e-05}
|
| 134 |
+
{"step": 1330, "total_steps": 28610, "tokens": 2789212160, "source": "fineweb_hq", "phase": 1, "loss": 3.664066404104233, "ce_loss": 3.646458864212036, "logit_z_loss": 12.97669005393982, "router_aux_loss": 1.5600038170814514, "router_z_loss": 0.7098991051316261, "routing_entropy_normalized": 0.7285120291314378, "learning_rate": 0.003, "grad_norm": 0.23489975929260254, "tokens_per_second": 3620163.3650206774, "eta_seconds": 15803.238912582396, "data_wait_seconds": 2.0623207092285156e-05}
|
| 135 |
+
{"step": 1340, "total_steps": 28610, "tokens": 2810183680, "source": "fineweb_hq", "phase": 1, "loss": 3.6751082241535187, "ce_loss": 3.657240390777588, "logit_z_loss": 13.218847632408142, "router_aux_loss": 1.5853353440761566, "router_z_loss": 0.6925933808088303, "routing_entropy_normalized": 0.7045790690481842, "learning_rate": 0.003, "grad_norm": 0.2663385570049286, "tokens_per_second": 3945755.5953272986, "eta_seconds": 14493.886825561523, "data_wait_seconds": 1.9043684005737305e-05}
|
| 136 |
+
{"step": 1350, "total_steps": 28610, "tokens": 2831155200, "source": "fineweb_edu", "phase": 1, "loss": 3.7320760786533356, "ce_loss": 3.7144225537776947, "logit_z_loss": 12.475766062736511, "router_aux_loss": 1.572474643588066, "router_z_loss": 0.6812893077731133, "routing_entropy_normalized": 0.7076111298507933, "learning_rate": 0.003, "grad_norm": 0.25115108489990234, "tokens_per_second": 3342246.1572490428, "eta_seconds": 17104.773505687714, "data_wait_seconds": 1.6123056411743164e-05}
|
| 137 |
+
{"step": 1360, "total_steps": 28610, "tokens": 2852126720, "source": "fineweb_hq", "phase": 1, "loss": 3.6170286536216736, "ce_loss": 3.599080264568329, "logit_z_loss": 12.926063179969788, "router_aux_loss": 1.5920685082674026, "router_z_loss": 0.7351187020540237, "routing_entropy_normalized": 0.7297136746099004, "learning_rate": 0.003, "grad_norm": 0.22807221114635468, "tokens_per_second": 3317722.4333629897, "eta_seconds": 17224.886393547058, "data_wait_seconds": 1.7464160919189453e-05}
|
| 138 |
+
{"step": 1370, "total_steps": 28610, "tokens": 2873098240, "source": "fineweb_edu", "phase": 1, "loss": 3.7073278725147247, "ce_loss": 3.689581722021103, "logit_z_loss": 12.86902379989624, "router_aux_loss": 1.5667932778596878, "router_z_loss": 0.79135512560606, "routing_entropy_normalized": 0.7026703592149608, "learning_rate": 0.003, "grad_norm": 0.22224768996238708, "tokens_per_second": 3848521.849059559, "eta_seconds": 14843.730325698853, "data_wait_seconds": 1.6868114471435547e-05}
|
| 139 |
+
{"step": 1380, "total_steps": 28610, "tokens": 2894069760, "source": "fineweb_hq", "phase": 1, "loss": 3.6396936178207397, "ce_loss": 3.62176114320755, "logit_z_loss": 13.100827813148499, "router_aux_loss": 1.582287460565567, "router_z_loss": 0.7995611429214478, "routing_entropy_normalized": 0.7139659143143634, "learning_rate": 0.003, "grad_norm": 0.25207850337028503, "tokens_per_second": 2669021.6950111184, "eta_seconds": 21395.648100852966, "data_wait_seconds": 1.9937753677368164e-05}
|
| 140 |
+
{"step": 1390, "total_steps": 28610, "tokens": 2915041280, "source": "fineweb_edu", "phase": 1, "loss": 3.712700366973877, "ce_loss": 3.695215582847595, "logit_z_loss": 12.642094135284424, "router_aux_loss": 1.5312270373106003, "router_z_loss": 0.9082884714007378, "routing_entropy_normalized": 0.7383512084120059, "learning_rate": 0.003, "grad_norm": 0.27023693919181824, "tokens_per_second": 3822487.99540611, "eta_seconds": 14933.853947639465, "data_wait_seconds": 2.250075340270996e-05}
|
| 141 |
+
{"step": 1400, "total_steps": 28610, "tokens": 2936012800, "source": "fineweb_hq", "phase": 1, "loss": 3.676722288131714, "ce_loss": 3.6587376296520233, "logit_z_loss": 12.360433101654053, "router_aux_loss": 1.6013674885034561, "router_z_loss": 0.734929546713829, "routing_entropy_normalized": 0.7145135967738396, "learning_rate": 0.003, "grad_norm": 0.2288474589586258, "tokens_per_second": 3615094.719415147, "eta_seconds": 15784.788601398468, "data_wait_seconds": 2.3037195205688477e-05}
|
| 142 |
+
{"step": 1410, "total_steps": 28610, "tokens": 2956984320, "source": "fineweb_hq", "phase": 1, "loss": 3.6780615150928497, "ce_loss": 3.6604828238487244, "logit_z_loss": 12.762844920158386, "router_aux_loss": 1.5506181418895721, "router_z_loss": 0.7961695119738579, "routing_entropy_normalized": 0.7289022059997208, "learning_rate": 0.003, "grad_norm": 0.21961157023906708, "tokens_per_second": 4164205.1542467577, "eta_seconds": 13698.300704956055, "data_wait_seconds": 2.1189451217651367e-05}
|
| 143 |
+
{"step": 1420, "total_steps": 28610, "tokens": 2977955840, "source": "fineweb_hq", "phase": 1, "loss": 3.6734668016433716, "ce_loss": 3.656163662672043, "logit_z_loss": 12.42779541015625, "router_aux_loss": 1.5250843912363052, "router_z_loss": 0.8095820993185043, "routing_entropy_normalized": 0.7271144684186339, "learning_rate": 0.003, "grad_norm": 0.22234903275966644, "tokens_per_second": 3781325.223634907, "eta_seconds": 15079.7827501297, "data_wait_seconds": 1.9341707229614258e-05}
|
| 144 |
+
{"step": 1430, "total_steps": 28610, "tokens": 2998927360, "source": "fineweb_hq", "phase": 1, "loss": 3.655671715736389, "ce_loss": 3.638202488422394, "logit_z_loss": 12.594285368919373, "router_aux_loss": 1.536058321595192, "router_z_loss": 0.8491563275456429, "routing_entropy_normalized": 0.7155607391324773, "learning_rate": 0.003, "grad_norm": 0.22061379253864288, "tokens_per_second": 3956496.72841663, "eta_seconds": 14406.83394241333, "data_wait_seconds": 1.8537044525146484e-05}
|