Training Step 5625

README.md

@@ -0,0 +1,10 @@
+---
+tags:
+- model_hub_mixin
+- pytorch_model_hub_mixin
+---
+
+This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:
+- Code: [More Information Needed]
+- Paper: [More Information Needed]
+- Docs: [More Information Needed]

config.json

@@ -0,0 +1,5 @@
+{
+  "embed_size": 32,
+  "hidden_size": 32,
+  "vocab_size": 50257
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:088f0e0957bc54fc551512a235b1231bb92b1aad1a0e2f8044462ab3a24c2646
+size 13101212

trainer_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eff8365666727017a6c01f58360902c1c3a87e9a885bccee32f3470f37598b11
+size 26209926

training_config.json

@@ -0,0 +1,98 @@
+{
+  "_hub_mixin_config": {
+    "optimizer_class": "adamw",
+    "scheduler_class": "cosine",
+    "input_files": null,
+    "dataset_subset_size": 1000000,
+    "block_size": 128,
+    "dataset_type": "fixed",
+    "train_ratio": 0.98,
+    "val_ratio": 0.01,
+    "encoding_name": "gpt2",
+    "n_heads": 12,
+    "n_blocks": 12,
+    "norm": "prenorm",
+    "pos_embed_type": "absolute",
+    "dropout": 0.1,
+    "embed_size": 32,
+    "hidden_size": 32,
+    "k": 3,
+    "num_inner_layers": 1,
+    "embedding_type": "full",
+    "checkpoint": null,
+    "batch_size": 64,
+    "num_epochs": 5,
+    "learning_rate": 0.0003,
+    "warmup_ratio": 0.1,
+    "log_interval_steps": 250,
+    "save_interval_steps": 5625,
+    "save_dir": "./saved_models/lstm_pico",
+    "save_latest": true,
+    "save_best": true,
+    "loss_metric_for_best_model": "val",
+    "prompt": "Once upon a",
+    "max_new_tokens": 50,
+    "top_p": 0.9,
+    "use_wandb": true,
+    "wandb_entity": "pico-llm",
+    "wandb_project": "training",
+    "wandb_name": "lstm-pico",
+    "upload_model_to_hub": true,
+    "repo_id": "pico-llm/lstm-pico",
+    "device": "cuda:0",
+    "seed": 42,
+    "monosemantic_analysis": true,
+    "num_steps": 76565
+  },
+  "hf_api": "<huggingface_hub.hf_api.HfApi object at 0x153dd3661040>",
+  "wandb_writer": "<wandb.sdk.wandb_run.Run object at 0x153dd36ad940>",
+  "wandb_table": "<wandb.sdk.data_types.table.Table object at 0x153dd36ae900>",
+  "optimizer": "AdamW (\nParameter Group 0\n    amsgrad: False\n    betas: (0.9, 0.999)\n    capturable: False\n    decoupled_weight_decay: True\n    differentiable: False\n    eps: 1e-08\n    foreach: None\n    fused: None\n    initial_lr: 0.0003\n    lr: 0.00022121120689655424\n    maximize: False\n    weight_decay: 0.05\n)",
+  "scheduler": "<torch.optim.lr_scheduler.SequentialLR object at 0x153e820e67b0>",
+  "optimizer_class": "adamw",
+  "scheduler_class": "cosine",
+  "model": "LSTMSeqModel(\n  (embedding): Embedding(50257, 32)\n  (lstm): LSTM(32, 32)\n  (linear): Linear(in_features=32, out_features=50257, bias=True)\n)",
+  "learning_rate": 0.0003,
+  "_init_kwargs": {
+    "input_files": null,
+    "dataset_subset_size": 1000000,
+    "block_size": 128,
+    "dataset_type": "fixed",
+    "train_ratio": 0.98,
+    "val_ratio": 0.01,
+    "encoding_name": "gpt2",
+    "n_heads": 12,
+    "n_blocks": 12,
+    "norm": "prenorm",
+    "pos_embed_type": "absolute",
+    "dropout": 0.1,
+    "embed_size": 32,
+    "hidden_size": 32,
+    "k": 3,
+    "num_inner_layers": 1,
+    "embedding_type": "full",
+    "checkpoint": null,
+    "batch_size": 64,
+    "num_epochs": 5,
+    "warmup_ratio": 0.1,
+    "log_interval_steps": 250,
+    "save_interval_steps": 5625,
+    "save_dir": "./saved_models/lstm_pico",
+    "save_latest": true,
+    "save_best": true,
+    "loss_metric_for_best_model": "val",
+    "prompt": "Once upon a",
+    "max_new_tokens": 50,
+    "top_p": 0.9,
+    "use_wandb": true,
+    "wandb_entity": "pico-llm",
+    "wandb_project": "training",
+    "wandb_name": "lstm-pico",
+    "upload_model_to_hub": true,
+    "repo_id": "pico-llm/lstm-pico",
+    "device": "cuda:0",
+    "seed": 42,
+    "monosemantic_analysis": true,
+    "num_steps": 76565
+  }
+}