florianvoss commited on
Commit
8b101c1
·
verified ·
1 Parent(s): b68bf9f

Remove runtime tokenizer truncation limit

Browse files

Set the runtime tokenizer truncation policy to null so prompts can use the compiled context. Compiled binaries and model data are unchanged.

Files changed (1) hide show
  1. devkit/tokenizer.json +2 -7
devkit/tokenizer.json CHANGED
@@ -1,11 +1,6 @@
1
  {
2
  "version": "1.0",
3
- "truncation": {
4
- "direction": "Right",
5
- "max_length": 2048,
6
- "strategy": "LongestFirst",
7
- "stride": 0
8
- },
9
  "padding": null,
10
  "added_tokens": [
11
  {
@@ -323814,4 +323809,4 @@
323814
  ]
323815
  ]
323816
  }
323817
- }
 
1
  {
2
  "version": "1.0",
3
+ "truncation": null,
 
 
 
 
 
4
  "padding": null,
5
  "added_tokens": [
6
  {
 
323809
  ]
323810
  ]
323811
  }
323812
+ }