fnet_tokenizer_fast / tokenizer.json
windowsartes's picture
Upload tokenizer
a9732d8 verified
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 906,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 907,
"content": "[SEP]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 908,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 909,
"content": "[CLS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 910,
"content": "[MASK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "BertNormalizer",
"clean_text": true,
"handle_chinese_chars": true,
"strip_accents": null,
"lowercase": false
},
"pre_tokenizer": {
"type": "BertPreTokenizer"
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"SpecialToken": {
"id": "[CLS]",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[SEP]",
"type_id": 0
}
}
],
"pair": [
{
"SpecialToken": {
"id": "[CLS]",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[SEP]",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "[SEP]",
"type_id": 1
}
}
],
"special_tokens": {
"[CLS]": {
"id": "[CLS]",
"ids": [
909
],
"tokens": [
"[CLS]"
]
},
"[SEP]": {
"id": "[SEP]",
"ids": [
907
],
"tokens": [
"[SEP]"
]
}
}
},
"decoder": {
"type": "WordPiece",
"prefix": "##",
"cleanup": true
},
"model": {
"type": "WordPiece",
"unk_token": "[UNK]",
"continuing_subword_prefix": "##",
"max_input_chars_per_word": 100,
"vocab": {
"G31": 0,
"F02": 1,
"E78": 2,
"G20": 3,
"K31": 4,
"K21": 5,
"K44": 6,
"F41": 7,
"I34": 8,
"M81": 9,
"G43": 10,
"I13": 11,
"I50": 12,
"E87": 13,
"E11": 14,
"I25": 15,
"D64": 16,
"G47": 17,
"M1A": 18,
"E66": 19,
"D50": 20,
"K29": 21,
"K25": 22,
"K57": 23,
"I12": 24,
"I70": 25,
"K22": 26,
"D63": 27,
"C67": 28,
"I10": 29,
"D25": 30,
"E89": 31,
"I26": 32,
"I82": 33,
"I47": 34,
"I87": 35,
"B95": 36,
"E03": 37,
"E83": 38,
"K59": 39,
"K65": 40,
"B96": 41,
"F17": 42,
"D72": 43,
"A41": 44,
"D62": 45,
"I27": 46,
"K43": 47,
"E43": 48,
"E53": 49,
"I48": 50,
"I11": 51,
"I45": 52,
"J44": 53,
"G56": 54,
"M19": 55,
"J45": 56,
"I73": 57,
"K72": 58,
"J96": 59,
"I44": 60,
"D69": 61,
"I46": 62,
"M47": 63,
"J01": 64,
"M79": 65,
"K92": 66,
"H40": 67,
"H69": 68,
"I24": 69,
"I49": 70,
"C16": 71,
"C78": 72,
"K91": 73,
"M86": 74,
"L97": 75,
"F32": 76,
"G25": 77,
"L03": 78,
"J98": 79,
"I96": 80,
"G89": 81,
"M25": 82,
"E46": 83,
"G30": 84,
"H91": 85,
"M17": 86,
"D68": 87,
"M46": 88,
"M70": 89,
"E55": 90,
"C50": 91,
"I83": 92,
"L76": 93,
"K76": 94,
"I67": 95,
"M51": 96,
"M48": 97,
"M35": 98,
"C79": 99,
"G93": 100,
"G91": 101,
"C34": 102,
"F05": 103,
"G51": 104,
"M21": 105,
"H53": 106,
"H04": 107,
"H35": 108,
"M50": 109,
"E04": 110,
"M80": 111,
"F03": 112,
"I28": 113,
"J43": 114,
"I07": 115,
"C43": 116,
"I65": 117,
"J18": 118,
"J90": 119,
"K40": 120,
"I30": 121,
"I31": 122,
"M06": 123,
"I21": 124,
"A04": 125,
"F44": 126,
"I95": 127,
"E86": 128,
"D51": 129,
"F43": 130,
"F50": 131,
"K02": 132,
"L73": 133,
"K85": 134,
"F39": 135,
"M54": 136,
"B20": 137,
"B19": 138,
"F10": 139,
"E16": 140,
"F19": 141,
"F14": 142,
"F20": 143,
"F29": 144,
"F11": 145,
"E10": 146,
"D86": 147,
"J02": 148,
"K20": 149,
"E21": 150,
"E51": 151,
"I69": 152,
"I63": 153,
"G81": 154,
"M85": 155,
"D49": 156,
"K74": 157,
"K66": 158,
"K62": 159,
"L91": 160,
"D57": 161,
"C90": 162,
"J91": 163,
"K56": 164,
"C21": 165,
"B37": 166,
"E88": 167,
"I35": 168,
"I42": 169,
"K61": 170,
"K60": 171,
"G92": 172,
"D46": 173,
"J84": 174,
"M84": 175,
"D18": 176,
"I61": 177,
"G40": 178,
"L02": 179,
"F88": 180,
"I08": 181,
"G62": 182,
"F31": 183,
"E44": 184,
"I15": 185,
"K80": 186,
"I16": 187,
"J69": 188,
"J95": 189,
"H26": 190,
"L40": 191,
"L57": 192,
"M72": 193,
"J30": 194,
"H61": 195,
"C56": 196,
"C77": 197,
"I81": 198,
"I85": 199,
"I86": 200,
"K70": 201,
"G96": 202,
"F12": 203,
"M53": 204,
"C7B": 205,
"C7A": 206,
"K58": 207,
"K26": 208,
"J81": 209,
"B18": 210,
"A08": 211,
"M96": 212,
"I97": 213,
"C22": 214,
"C25": 215,
"K86": 216,
"K52": 217,
"K11": 218,
"B00": 219,
"A40": 220,
"K83": 221,
"C24": 222,
"D53": 223,
"C20": 224,
"H90": 225,
"D47": 226,
"L29": 227,
"K81": 228,
"K82": 229,
"D70": 230,
"M10": 231,
"F90": 232,
"D13": 233,
"I71": 234,
"F33": 235,
"K75": 236,
"F22": 237,
"F15": 238,
"F72": 239,
"F13": 240,
"J36": 241,
"L27": 242,
"D61": 243,
"C92": 244,
"E85": 245,
"K12": 246,
"L99": 247,
"L72": 248,
"G58": 249,
"F45": 250,
"H66": 251,
"J11": 252,
"F60": 253,
"L30": 254,
"J86": 255,
"J15": 256,
"A81": 257,
"M40": 258,
"L08": 259,
"L60": 260,
"L82": 261,
"M62": 262,
"H93": 263,
"I72": 264,
"F42": 265,
"K04": 266,
"E80": 267,
"L53": 268,
"L89": 269,
"L98": 270,
"H81": 271,
"D80": 272,
"B35": 273,
"C73": 274,
"E05": 275,
"J39": 276,
"I22": 277,
"I51": 278,
"C18": 279,
"C91": 280,
"C62": 281,
"K64": 282,
"K63": 283,
"E08": 284,
"F09": 285,
"K28": 286,
"E22": 287,
"F25": 288,
"M94": 289,
"M71": 290,
"J20": 291,
"K95": 292,
"M16": 293,
"L85": 294,
"B34": 295,
"J04": 296,
"J47": 297,
"M31": 298,
"M41": 299,
"L71": 300,
"M00": 301,
"M89": 302,
"J94": 303,
"C61": 304,
"C80": 305,
"J93": 306,
"K50": 307,
"B02": 308,
"J06": 309,
"I09": 310,
"D32": 311,
"I33": 312,
"L28": 313,
"F23": 314,
"G12": 315,
"E02": 316,
"I60": 317,
"L95": 318,
"H33": 319,
"K08": 320,
"D73": 321,
"I20": 322,
"D59": 323,
"E79": 324,
"J60": 325,
"C95": 326,
"C94": 327,
"F84": 328,
"L05": 329,
"M30": 330,
"J32": 331,
"D23": 332,
"K42": 333,
"D35": 334,
"J34": 335,
"E26": 336,
"I80": 337,
"K51": 338,
"I36": 339,
"K55": 340,
"E27": 341,
"D75": 342,
"D55": 343,
"J40": 344,
"G90": 345,
"D12": 346,
"E56": 347,
"F51": 348,
"G35": 349,
"M87": 350,
"E06": 351,
"I43": 352,
"G44": 353,
"G95": 354,
"C02": 355,
"G83": 356,
"F79": 357,
"I77": 358,
"E73": 359,
"H05": 360,
"H54": 361,
"H46": 362,
"M08": 363,
"M11": 364,
"H11": 365,
"F28": 366,
"M97": 367,
"C60": 368,
"M99": 369,
"M34": 370,
"F01": 371,
"I66": 372,
"H43": 373,
"H25": 374,
"D89": 375,
"G60": 376,
"I89": 377,
"M20": 378,
"E13": 379,
"L93": 380,
"E61": 381,
"A09": 382,
"G24": 383,
"G21": 384,
"M75": 385,
"M05": 386,
"D60": 387,
"B44": 388,
"B59": 389,
"A49": 390,
"A18": 391,
"M43": 392,
"M23": 393,
"K94": 394,
"C81": 395,
"K35": 396,
"F07": 397,
"G82": 398,
"E74": 399,
"C45": 400,
"E59": 401,
"E60": 402,
"A48": 403,
"G45": 404,
"F30": 405,
"H34": 406,
"C85": 407,
"M32": 408,
"I05": 409,
"D28": 410,
"M24": 411,
"E23": 412,
"C71": 413,
"F40": 414,
"D38": 415,
"D48": 416,
"C64": 417,
"H10": 418,
"D65": 419,
"B49": 420,
"H16": 421,
"I99": 422,
"L70": 423,
"C48": 424,
"D43": 425,
"E07": 426,
"L10": 427,
"C83": 428,
"J70": 429,
"I68": 430,
"J99": 431,
"H15": 432,
"B36": 433,
"D58": 434,
"L23": 435,
"I23": 436,
"J09": 437,
"B94": 438,
"I74": 439,
"G54": 440,
"B99": 441,
"G72": 442,
"G50": 443,
"G97": 444,
"D17": 445,
"E09": 446,
"L13": 447,
"G61": 448,
"A43": 449,
"H02": 450,
"H57": 451,
"H49": 452,
"I38": 453,
"G37": 454,
"F21": 455,
"D56": 456,
"K90": 457,
"J92": 458,
"J12": 459,
"B97": 460,
"J38": 461,
"L80": 462,
"H31": 463,
"A42": 464,
"M13": 465,
"M77": 466,
"M65": 467,
"L88": 468,
"C44": 469,
"G06": 470,
"L25": 471,
"D34": 472,
"K46": 473,
"M60": 474,
"I62": 475,
"D83": 476,
"F64": 477,
"D41": 478,
"E72": 479,
"B60": 480,
"J61": 481,
"J85": 482,
"C10": 483,
"A63": 484,
"C14": 485,
"C09": 486,
"F55": 487,
"F99": 488,
"M76": 489,
"A60": 490,
"C54": 491,
"F91": 492,
"F63": 493,
"J14": 494,
"F06": 495,
"C88": 496,
"H83": 497,
"J31": 498,
"I37": 499,
"K68": 500,
"B45": 501,
"G04": 502,
"J10": 503,
"B85": 504,
"C46": 505,
"B48": 506,
"C40": 507,
"J80": 508,
"H51": 509,
"G46": 510,
"M15": 511,
"K06": 512,
"A69": 513,
"J13": 514,
"I78": 515,
"J00": 516,
"M27": 517,
"G08": 518,
"H92": 519,
"E63": 520,
"B33": 521,
"C19": 522,
"E34": 523,
"L50": 524,
"I06": 525,
"K71": 526,
"K36": 527,
"K14": 528,
"H18": 529,
"E29": 530,
"E54": 531,
"D30": 532,
"E28": 533,
"K30": 534,
"B86": 535,
"G57": 536,
"H21": 537,
"H44": 538,
"H20": 539,
"C15": 540,
"G80": 541,
"D36": 542,
"F34": 543,
"D27": 544,
"C51": 545,
"L90": 546,
"D52": 547,
"H55": 548,
"J62": 549,
"C93": 550,
"C57": 551,
"D3A": 552,
"A03": 553,
"E84": 554,
"D45": 555,
"K03": 556,
"A31": 557,
"B27": 558,
"L81": 559,
"G63": 560,
"D67": 561,
"F16": 562,
"K41": 563,
"K05": 564,
"G23": 565,
"C41": 566,
"K73": 567,
"J33": 568,
"L21": 569,
"D76": 570,
"E31": 571,
"C82": 572,
"B25": 573,
"H30": 574,
"A07": 575,
"B17": 576,
"L64": 577,
"K27": 578,
"L24": 579,
"M45": 580,
"F48": 581,
"A77": 582,
"C49": 583,
"G71": 584,
"A15": 585,
"L54": 586,
"C23": 587,
"G99": 588,
"H60": 589,
"M12": 590,
"F81": 591,
"M90": 592,
"H36": 593,
"D15": 594,
"C68": 595,
"L74": 596,
"C65": 597,
"A53": 598,
"M95": 599,
"E65": 600,
"C4A": 601,
"H42": 602,
"H50": 603,
"M88": 604,
"H72": 605,
"L94": 606,
"A19": 607,
"C76": 608,
"L55": 609,
"D71": 610,
"D40": 611,
"G70": 612,
"G52": 613,
"L43": 614,
"E24": 615,
"C52": 616,
"L20": 617,
"B82": 618,
"C37": 619,
"D05": 620,
"J42": 621,
"C53": 622,
"J82": 623,
"H70": 624,
"C74": 625,
"D22": 626,
"B15": 627,
"D02": 628,
"B30": 629,
"C17": 630,
"I76": 631,
"B91": 632,
"C84": 633,
"B78": 634,
"L01": 635,
"E20": 636,
"M92": 637,
"J67": 638,
"J05": 639,
"K13": 640,
"A46": 641,
"G03": 642,
"B09": 643,
"F95": 644,
"C26": 645,
"F53": 646,
"C06": 647,
"F18": 648,
"C72": 649,
"D33": 650,
"A87": 651,
"H47": 652,
"I32": 653,
"G59": 654,
"B16": 655,
"K37": 656,
"H01": 657,
"B01": 658,
"G98": 659,
"D44": 660,
"E50": 661,
"H80": 662,
"G36": 663,
"D74": 664,
"M66": 665,
"C86": 666,
"C32": 667,
"C66": 668,
"L92": 669,
"G65": 670,
"D26": 671,
"F70": 672,
"M26": 673,
"C33": 674,
"K01": 675,
"M18": 676,
"D00": 677,
"B46": 678,
"D37": 679,
"A02": 680,
"C07": 681,
"F71": 682,
"G10": 683,
"H27": 684,
"L63": 685,
"K38": 686,
"C04": 687,
"C08": 688,
"J16": 689,
"H52": 690,
"L65": 691,
"L84": 692,
"D16": 693,
"J03": 694,
"L52": 695,
"G00": 696,
"I75": 697,
"H00": 698,
"L11": 699,
"E01": 700,
"H59": 701,
"M22": 702,
"C96": 703,
"B07": 704,
"D84": 705,
"E32": 706,
"A54": 707,
"A56": 708,
"D39": 709,
"E70": 710,
"M14": 711,
"A52": 712,
"F54": 713,
"F98": 714,
"F68": 715,
"A86": 716,
"A64": 717,
"M61": 718,
"I88": 719,
"D29": 720,
"J22": 721,
"F80": 722,
"D24": 723,
"B53": 724,
"J66": 725,
"F78": 726,
"E75": 727,
"M83": 728,
"B90": 729,
"A44": 730,
"E41": 731,
"M33": 732,
"L59": 733,
"M67": 734,
"D04": 735,
"D09": 736,
"K45": 737,
"D07": 738,
"D01": 739,
"M93": 740,
"D78": 741,
"L12": 742,
"I40": 743,
"J68": 744,
"G11": 745,
"L68": 746,
"M07": 747,
"A36": 748,
"F82": 749,
"C05": 750,
"J35": 751,
"F89": 752,
"J21": 753,
"E71": 754,
"G14": 755,
"C70": 756,
"B10": 757,
"C58": 758,
"C01": 759,
"L44": 760,
"C55": 761,
"G13": 762,
"C13": 763,
"A28": 764,
"D66": 765,
"M36": 766,
"B88": 767,
"B47": 768,
"A74": 769,
"B58": 770,
"I00": 771,
"A85": 772,
"L56": 773,
"F04": 774,
"G94": 775,
"A39": 776,
"D21": 777,
"B57": 778,
"B69": 779,
"L04": 780,
"L51": 781,
"M02": 782,
"J17": 783,
"D82": 784,
"C47": 785,
"K00": 786,
"H73": 787,
"J63": 788,
"A51": 789,
"F65": 790,
"F69": 791,
"F94": 792,
"J41": 793,
"D31": 794,
"B67": 795,
"M01": 796,
"G73": 797,
"D19": 798,
"C12": 799,
"D11": 800,
"A93": 801,
"B65": 802,
"L86": 803,
"B87": 804,
"C75": 805,
"C03": 806,
"E00": 807,
"H71": 808,
"C31": 809,
"G32": 810,
"L83": 811,
"H74": 812,
"A05": 813,
"B50": 814,
"D03": 815,
"G09": 816,
"B54": 817,
"E25": 818,
"D06": 819,
"C11": 820,
"A21": 821,
"M04": 822,
"B08": 823,
"D14": 824,
"A59": 825,
"D10": 826,
"A75": 827,
"A32": 828,
"C00": 829,
"M63": 830,
"I39": 831,
"B83": 832,
"B26": 833,
"E64": 834,
"G53": 835,
"A90": 836,
"E77": 837,
"A70": 838,
"G55": 839,
"C38": 840,
"A84": 841,
"E52": 842,
"F73": 843,
"I01": 844,
"A22": 845,
"H95": 846,
"F52": 847,
"G64": 848,
"J37": 849,
"A37": 850,
"D81": 851,
"E58": 852,
"G05": 853,
"H65": 854,
"A01": 855,
"A26": 856,
"E67": 857,
"B38": 858,
"L87": 859,
"M42": 860,
"H62": 861,
"L75": 862,
"A35": 863,
"L22": 864,
"C63": 865,
"A79": 866,
"K87": 867,
"M91": 868,
"L66": 869,
"B43": 870,
"B40": 871,
"B42": 872,
"L42": 873,
"A24": 874,
"K77": 875,
"G26": 876,
"E68": 877,
"C69": 878,
"F59": 879,
"H75": 880,
"H94": 881,
"H67": 882,
"L26": 883,
"C30": 884,
"H68": 885,
"A55": 886,
"A57": 887,
"J64": 888,
"A50": 889,
"B76": 890,
"H17": 891,
"A06": 892,
"E76": 893,
"A17": 894,
"L41": 895,
"B39": 896,
"D42": 897,
"A92": 898,
"A83": 899,
"D20": 900,
"L58": 901,
"B56": 902,
"A27": 903,
"A80": 904,
"A68": 905
}
}
}