add tokenizer

special_tokens_map.json

@@ -1 +1 @@
-{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}
+{}

tokenizer.json

@@ -2,7 +2,7 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 1000,
+    "max_length": 512,
     "strategy": "LongestFirst",
     "stride": 0
   },
@@ -66,7 +66,7 @@
     "clean_text": true,
     "handle_chinese_chars": true,
     "strip_accents": null,
-    "lowercase": true
+    "lowercase": false
   },
   "pre_tokenizer": {
     "type": "WhitespaceSplit"
@@ -183,17 +183,17 @@
       "r": 56,
       "s": 57,
       "##\"": 58,
-      "##N": 59,
-      "##B": 60,
-      "##[": 61,
-      "##F": 62,
-      "##O": 63,
-      "##S": 64,
-      "##E": 65,
-      "##P": 66,
-      "##]": 67,
-      "##c": 68,
-      "##C": 69
+      "##S": 59,
+      "##E": 60,
+      "##P": 61,
+      "##]": 62,
+      "##C": 63,
+      "##O": 64,
+      "##F": 65,
+      "##[": 66,
+      "##N": 67,
+      "##B": 68,
+      "##c": 69
     }
   }
 }

tokenizer_config.json

@@ -1 +1 @@
-{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "tokenizer_class": "BertTokenizer"}
+{"tokenizer_class": "PreTrainedTokenizerFast"}