Upload OGBERT tokenizer (vocab_size=16384)

README.md

@@ -1,25 +1,49 @@
 ---
-library_name: tokenizers
-pipeline_tag: feature-extraction
 language:
 - en
-license: mit
+license: apache-2.0
+library_name: transformers
 tags:
+- tokenizer
+- bpe
 - ogbert
 - modernbert
 - opengloss
-- tokenizer
-- bpe
-- vocab:16384
-datasets:
-- mjbommar/opengloss-v1.1-dictionary
 ---
 
-# OGBERT Tokenizer (16384)
+# OGBERT Tokenizer (16K)
+
+A 16,384-token BPE tokenizer for [OpenGloss](https://arxiv.org/abs/2511.18622) OGBERT embedding models.
+
+## Usage
+
+```python
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("mjbommar/ogbert-tokenizer-16k")
+tokens = tokenizer.encode("hello world")
+```
+
+## Details
+
+- **Vocab Size**: 16,384 (power of 2)
+- **Space Token**: ID 16383
+- **Special Tokens**: IDs 0-6 (`<|start|>`, `<|end|>`, `<|pad|>`, `<|unk|>`, `<|cls|>`, `<|sep|>`, `<|mask|>`)
+- **Training Data**: [mjbommar/opengloss-v1.1-dictionary](https://huggingface.co/datasets/mjbommar/opengloss-v1.1-dictionary)
+
+## Citation
+
+```bibtex
+@misc{bommarito2025opengloss,
+    title={OpenGloss: A Synthetic Encyclopedic Dictionary and Semantic Knowledge Graph},
+    author={Michael J. Bommarito II},
+    year={2025},
+    eprint={2511.18622},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+```
 
-Byte-level BPE tokenizer for OGBERT models. Trained on OpenGloss headwords only, with ordered specials (<|start|>, <|end|>, <|pad|>, <|unk|>, <|cls|>, <|sep|>, <|mask|>) and a final non-special space token that does not participate in merges. Suitable for ModernBERT/transformers usage.
+## License
 
-- Vocab size: 16384
-- Alphabet: 0-255 bytes + specials + trailing space token
-- Training data: OpenGloss dictionary headwords (HF dataset mjbommar/opengloss-v1.1-dictionary)
-- Notes: space token is appended to avoid merges; special tokens are in fixed order.
+Apache 2.0

tokenizer.json

@@ -67,7 +67,7 @@
       "special": true
     },
     {
-      "id": 16384,
+      "id": 16383,
       "content": " ",
       "single_word": false,
       "lstrip": false,
@@ -16481,8 +16481,7 @@
       "propriet": 16379,
       "adventure": 16380,
       "shorter": 16381,
-      "shorts": 16382,
-      "nikola": 16383
+      "shorts": 16382
     },
     "merges": [
       [
@@ -80964,10 +80963,6 @@
       [
         "shor",
         "ts"
-      ],
-      [
-        "nik",
-        "ola"
       ]
     ]
   }

tokenizer_config.json

@@ -1,22 +1,16 @@
 {
-  "tokenizer_class": "PreTrainedTokenizerFast",
+  "additional_special_tokens": null,
+  "backend": "tokenizers",
   "bos_token": "<|start|>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<|cls|>",
   "eos_token": "<|end|>",
+  "mask_token": "<|mask|>",
+  "model_max_length": 1024,
   "pad_token": "<|pad|>",
-  "unk_token": "<|unk|>",
-  "cls_token": "<|cls|>",
   "sep_token": "<|sep|>",
-  "mask_token": "<|mask|>",
-  "model_max_length": 4096,
-  "padding_side": "right",
-  "truncation": "longest_first",
-  "special_tokens_map": {
-    "bos_token": "<|start|>",
-    "eos_token": "<|end|>",
-    "pad_token": "<|pad|>",
-    "unk_token": "<|unk|>",
-    "cls_token": "<|cls|>",
-    "sep_token": "<|sep|>",
-    "mask_token": "<|mask|>"
-  }
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<|unk|>",
+  "model_type": "modernbert",
+  "vocab_size": 16384
 }