Upload tokenizer.json with huggingface_hub

tokenizer.json

@@ -31,7 +31,43 @@
       "special": true
     },
     {
-      "id": 32000,
+      "id": 32007,
+      "content": "▁<PRE>",
+      "single_word": false,
+      "lstrip": true,
+      "rstrip": true,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32008,
+      "content": "▁<SUF>",
+      "single_word": false,
+      "lstrip": true,
+      "rstrip": true,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32009,
+      "content": "▁<MID>",
+      "single_word": false,
+      "lstrip": true,
+      "rstrip": true,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32010,
+      "content": "▁<EOT>",
+      "single_word": false,
+      "lstrip": true,
+      "rstrip": true,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32016,
       "content": "<PAD>",
       "single_word": false,
       "lstrip": false,
@@ -32143,7 +32179,23 @@
       "왕": 31996,
       "收": 31997,
       "弘": 31998,
-      "给": 31999
+      "给": 31999,
+      "▁<SU": 32000,
+      "▁<SUF": 32001,
+      "▁<PRE": 32002,
+      "▁<M": 32003,
+      "▁<MID": 32004,
+      "▁<E": 32005,
+      "▁<EOT": 32006,
+      "▁<PRE>": 32007,
+      "▁<SUF>": 32008,
+      "▁<MID>": 32009,
+      "▁<EOT>": 32010,
+      "▁<EOT><EOT>": 32011,
+      "▁<EOT><EOT><EOT>": 32012,
+      "▁<EOT><EOT><EOT><EOT>": 32013,
+      "▁<EOT><EOT><EOT><EOT><EOT>": 32014,
+      "▁<EOT><EOT><EOT><EOT><EOT><EOT>": 32015
     },
     "merges": [
       "▁ t",
@@ -93394,7 +93446,18 @@
       "▁▁▁▁▁▁▁▁▁ ▁▁▁▁▁▁",
       "▁▁▁▁▁▁▁ ▁▁▁▁▁▁▁▁",
       "▁▁▁▁▁▁▁▁▁▁▁ ▁▁▁▁",
-      "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
+      "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "▁< SU",
+      "▁<SU F",
+      "▁< PRE",
+      "▁< M",
+      "▁<M ID",
+      "▁< E",
+      "▁<E OT",
+      "▁<PRE >",
+      "▁<SUF >",
+      "▁<MID >",
+      "▁<EOT >"
     ]
   }
 }