add tokenizer
Browse files- tokenizer.json +12 -12
- vocab.txt +5 -5
tokenizer.json
CHANGED
|
@@ -185,18 +185,18 @@
|
|
| 185 |
"o": 55,
|
| 186 |
"r": 56,
|
| 187 |
"s": 57,
|
| 188 |
-
"
|
| 189 |
-
"
|
| 190 |
-
"##
|
| 191 |
-
"##
|
| 192 |
-
"##
|
| 193 |
-
"##
|
| 194 |
-
"##
|
| 195 |
-
"##
|
| 196 |
-
"##
|
| 197 |
-
"##
|
| 198 |
-
"##
|
| 199 |
-
"##
|
| 200 |
}
|
| 201 |
}
|
| 202 |
}
|
|
|
|
| 185 |
"o": 55,
|
| 186 |
"r": 56,
|
| 187 |
"s": 57,
|
| 188 |
+
"##\"": 58,
|
| 189 |
+
"##N": 59,
|
| 190 |
+
"##F": 60,
|
| 191 |
+
"##[": 61,
|
| 192 |
+
"##S": 62,
|
| 193 |
+
"##E": 63,
|
| 194 |
+
"##P": 64,
|
| 195 |
+
"##]": 65,
|
| 196 |
+
"##O": 66,
|
| 197 |
+
"##c": 67,
|
| 198 |
+
"##C": 68,
|
| 199 |
+
"##B": 69
|
| 200 |
}
|
| 201 |
}
|
| 202 |
}
|
vocab.txt
CHANGED
|
@@ -56,15 +56,15 @@ n
|
|
| 56 |
o
|
| 57 |
r
|
| 58 |
s
|
| 59 |
-
##O
|
| 60 |
##"
|
|
|
|
|
|
|
|
|
|
| 61 |
##S
|
| 62 |
##E
|
| 63 |
##P
|
| 64 |
##]
|
|
|
|
|
|
|
| 65 |
##C
|
| 66 |
-
##N
|
| 67 |
##B
|
| 68 |
-
##[
|
| 69 |
-
##c
|
| 70 |
-
##F
|
|
|
|
| 56 |
o
|
| 57 |
r
|
| 58 |
s
|
|
|
|
| 59 |
##"
|
| 60 |
+
##N
|
| 61 |
+
##F
|
| 62 |
+
##[
|
| 63 |
##S
|
| 64 |
##E
|
| 65 |
##P
|
| 66 |
##]
|
| 67 |
+
##O
|
| 68 |
+
##c
|
| 69 |
##C
|
|
|
|
| 70 |
##B
|
|
|
|
|
|
|
|
|