Upload tokenizer.json with huggingface_hub

tokenizer.json

@@ -0,0 +1,270 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": {
+    "strategy": "BatchLongest",
+    "direction": "Right",
+    "pad_to_multiple_of": null,
+    "pad_id": 0,
+    "pad_type_id": 0,
+    "pad_token": "[PAD]"
+  },
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[BOS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[EOS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "[PAD]": 0,
+      "[UNK]": 1,
+      "[BOS]": 2,
+      "[EOS]": 3,
+      "the": 4,
+      "|>": 5,
+      "'": 6,
+      "(": 7,
+      "Keys": 8,
+      "database": 9,
+      "end": 10,
+      ",": 11,
+      ")": 12,
+      "',": 13,
+      "above": 14,
+      "and": 15,
+      "for": 16,
+      "that": 17,
+      ").": 18,
+      ".": 19,
+      ":": 20,
+      "<|": 21,
+      "FK": 22,
+      "FROM": 23,
+      "Foreign": 24,
+      "Given": 25,
+      "PK": 26,
+      "Primary": 27,
+      "SELECT": 28,
+      "SQL": 29,
+      "Then": 30,
+      "analyze": 31,
+      "answers": 32,
+      "based": 33,
+      "between": 34,
+      "displayed": 35,
+      "on": 36,
+      "query": 37,
+      "question": 38,
+      "relationships": 39,
+      "schema": 40,
+      "system": 41,
+      "tables": 42,
+      "user": 43,
+      "write": 44,
+      "|><|": 45,
+      "WHERE": 46,
+      ";<|": 47,
+      ".'<|": 48,
+      "=": 49,
+      "books": 50,
+      "?'<|": 51,
+      "BY": 52,
+      "GROUP": 53,
+      "loans": 54,
+      "priority": 55,
+      "users": 56,
+      "(*)": 57,
+      "-": 58,
+      "01": 59,
+      "15": 60,
+      "2024": 61,
+      "3": 62,
+      "5": 63,
+      ">": 64,
+      "AS": 65,
+      "COUNT": 66,
+      "JOIN": 67,
+      "List": 68,
+      "ON": 69,
+      "SUM": 70,
+      "amount": 71,
+      "customer_id": 72,
+      "date": 73,
+      "employees": 74,
+      "id": 75,
+      "month": 76,
+      "name": 77,
+      "open": 78,
+      "products": 79,
+      "revenue": 80,
+      "students": 81,
+      "');<|": 82,
+      "';<|": 83,
+      "('": 84,
+      "('%": 85,
+      "-%'": 86,
+      "0": 87,
+      "2025": 88,
+      "<": 89,
+      ">=": 90,
+      "Count": 91,
+      "DATE": 92,
+      "Find": 93,
+      "GPA": 94,
+      "How": 95,
+      "LIKE": 96,
+      "Show": 97,
+      "What": 98,
+      "Which": 99,
+      "are": 100,
+      "book_id": 101,
+      "borrowers": 102,
+      "by": 103,
+      "customer": 104,
+      "delay_minutes": 105,
+      "delays": 106,
+      "due_date": 107,
+      "expense": 108,
+      "expenses": 109,
+      "gpa": 110,
+      "have": 111,
+      "hire_date": 112,
+      "is": 113,
+      "joined": 114,
+      "m": 115,
+      "many": 116,
+      "minutes": 117,
+      "monthly": 118,
+      "now": 119,
+      "of": 120,
+      "orders": 121,
+      "out": 122,
+      "over": 123,
+      "overdue": 124,
+      "per": 125,
+      "product_name": 126,
+      "quantity": 127,
+      "route_id": 128,
+      "routes": 129,
+      "status": 130,
+      "stock": 131,
+      "strftime": 132,
+      "support_tickets": 133,
+      "their": 134,
+      "this": 135,
+      "tickets": 136,
+      "title": 137,
+      "total": 138,
+      "totals": 139,
+      "trips": 140,
+      "user_id": 141,
+      "with": 142,
+      "year": 143,
+      "education_11": 144,
+      "education_19": 145,
+      "education_27": 146,
+      "education_3": 147,
+      "education_35": 148,
+      "education_43": 149,
+      "education_51": 150,
+      "education_59": 151,
+      "finance_12": 152,
+      "finance_20": 153,
+      "finance_28": 154,
+      "finance_36": 155,
+      "finance_4": 156,
+      "finance_44": 157,
+      "finance_52": 158,
+      "finance_60": 159,
+      "hr_10": 160,
+      "hr_18": 161,
+      "hr_2": 162,
+      "hr_26": 163,
+      "hr_34": 164,
+      "hr_42": 165,
+      "hr_50": 166,
+      "hr_58": 167,
+      "inventory_1": 168,
+      "inventory_17": 169,
+      "inventory_25": 170,
+      "inventory_33": 171,
+      "inventory_41": 172,
+      "inventory_49": 173,
+      "inventory_57": 174,
+      "inventory_9": 175,
+      "library_14": 176,
+      "library_22": 177,
+      "library_30": 178,
+      "library_38": 179,
+      "library_46": 180,
+      "library_54": 181,
+      "library_6": 182,
+      "library_62": 183,
+      "sales_0": 184,
+      "sales_16": 185,
+      "sales_24": 186,
+      "sales_32": 187,
+      "sales_40": 188,
+      "sales_48": 189,
+      "sales_56": 190,
+      "sales_8": 191,
+      "tickets_15": 192,
+      "tickets_23": 193,
+      "tickets_31": 194,
+      "tickets_39": 195,
+      "tickets_47": 196,
+      "tickets_55": 197,
+      "tickets_63": 198,
+      "tickets_7": 199,
+      "transport_13": 200,
+      "transport_21": 201,
+      "transport_29": 202,
+      "transport_37": 203,
+      "transport_45": 204,
+      "transport_5": 205,
+      "transport_53": 206,
+      "transport_61": 207
+    },
+    "unk_token": "[UNK]"
+  }
+}