{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[SOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[EOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "[MSK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 5, "content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": { "type": "Sequence", "normalizers": [ { "type": "NFD" }, { "type": "StripAccents" }, { "type": "Strip", "strip_left": true, "strip_right": true } ] }, "pre_tokenizer": { "type": "Whitespace" }, "post_processor": null, "decoder": null, "model": { "type": "WordLevel", "vocab": { "[PAD]": 0, "[SOS]": 1, "[EOS]": 2, "[UNK]": 3, "[MSK]": 4, "[SEP]": 5, "expression_statement__": 6, "__expression_statement": 7, "assignment": 8, "__if_statement": 9, "if_statement__": 10, "expression_statement": 11, "return_statement": 12, "__parenthesized_expression": 13, "parenthesized_expression__": 14, "binary_expression": 15, "binary_expression__": 16, "__binary_expression": 17, "__call_expression": 18, "call_expression__": 19, "assignment_expression__": 20, "__assignment_expression": 21, "member_expression": 22, "selector_expression": 23, "compound_statement__": 24, "__compound_statement": 25, "local_variable_declaration": 26, "return_statement__": 27, "__return_statement": 28, "for_statement__": 29, "__for_statement": 30, "assignment_expression": 31, "member_access_expression": 32, "object_creation_expression": 33, "call_statement": 34, "member_call_expression": 35, "subscript_expression": 36, "local_variable_declaration__": 37, "__local_variable_declaration": 38, "member_call_expression__": 39, "__member_call_expression": 40, "parenthesized_expression": 41, "__assignment_statement": 42, "assignment_statement__": 43, "try_statement__": 44, "__try_statement": 45, "call_expression": 46, "unary_expression": 47, "function_call_expression": 48, "raise_statement": 49, "__member_expression": 50, "member_expression__": 51, "function_call_expression__": 52, "__function_call_expression": 53, "__short_var_declaration": 54, "short_var_declaration__": 55, "subscript_expression__": 56, "__subscript_expression": 57, "call_statement__": 58, "__call_statement": 59, "array_creation_expression": 60, "selector_expression__": 61, "__selector_expression": 62, "__throw_statement": 63, "throw_statement__": 64, "cast_expression": 65, "update_expression": 66, "object_creation_expression__": 67, "__object_creation_expression": 68, "augmented_assignment": 69, "unary_expression__": 70, "__unary_expression": 71, "break_statement": 72, "__unary_op_expression": 73, "unary_op_expression__": 74, "enhanced_for_statement__": 75, "__enhanced_for_statement": 76, "scoped_call_expression": 77, "class_constant_access_expression": 78, "assignment_statement": 79, "while_statement__": 80, "__while_statement": 81, "if_statement": 82, "continue_statement": 83, "ternary_expression__": 84, "__ternary_expression": 85, "__foreach_statement": 86, "foreach_statement__": 87, "with_statement__": 88, "__with_statement": 89, "array_creation_expression__": 90, "__array_creation_expression": 91, "instanceof_expression": 92, "throw_expression__": 93, "__throw_expression": 94, "assert_statement": 95, "conditional_expression__": 96, "__conditional_expression": 97, "__scoped_call_expression": 98, "scoped_call_expression__": 99, "index_expression": 100, "__member_access_expression": 101, "member_access_expression__": 102, "cast_expression__": 103, "__cast_expression": 104, "index_expression__": 105, "__index_expression": 106, "__augmented_assignment_expression": 107, "augmented_assignment_expression__": 108, "pass_statement": 109, "scoped_property_access_expression": 110, "new_expression": 111, "import_from_statement": 112, "unless_statement": 113, "var_declaration": 114, "else_statement": 115, "__else_statement": 116, "else_statement__": 117, "method_declaration__": 118, "__method_declaration": 119, "new_expression__": 120, "__new_expression": 121, "unary_op_expression": 122, "short_var_declaration": 123, "__case_statement": 124, "case_statement__": 125, "__anonymous_function_creation_expression": 126, "anonymous_function_creation_expression__": 127, "augmented_assignment_expression": 128, "__defer_statement": 129, "defer_statement__": 130, "delete_statement": 131, "throw_statement": 132, "parameter_declaration": 133, "import_statement": 134, "when_statement": 135, "when_statement__": 136, "__when_statement": 137, "__lambda_expression": 138, "lambda_expression__": 139, "ternary_expression": 140, "switch_expression__": 141, "__switch_expression": 142, "__synchronized_statement": 143, "synchronized_statement__": 144, "print_statement": 145, "lambda_expression": 146, "switch_statement__": 147, "__switch_statement": 148, "operator_assignment_statement": 149, "type_assertion_expression__": 150, "__type_assertion_expression": 151, "rescue_statement": 152, "__assert_statement": 153, "assert_statement__": 154, "for_in_statement__": 155, "__for_in_statement": 156, "__rescue_statement": 157, "rescue_statement__": 158, "__unset_statement": 159, "unset_statement__": 160, "type_assertion_expression": 161, "__assignment": 162, "assignment__": 163, "__unless_statement": 164, "unless_statement__": 165, "expression_switch_statement__": 166, "__expression_switch_statement": 167, "inc_statement": 168, "slice_expression": 169, "__slice_expression": 170, "slice_expression__": 171, "__operator_assignment_statement": 172, "operator_assignment_statement__": 173, "__try_with_resources_statement": 174, "try_with_resources_statement__": 175, "global_statement": 176, "receive_statement__": 177, "__receive_statement": 178, "field_declaration": 179, "await_expression__": 180, "__await_expression": 181, "type_switch_statement__": 182, "__type_switch_statement": 183, "do_statement__": 184, "__do_statement": 185, "__go_statement": 186, "go_statement__": 187, "case_statement": 188, "type_conversion_expression": 189, "sequence_expression__": 190, "__sequence_expression": 191, "__update_expression": 192, "update_expression__": 193, "default_statement__": 194, "__default_statement": 195, "select_statement__": 196, "__select_statement": 197, "conditional_expression": 198, "__send_statement": 199, "send_statement__": 200, "__echo_statement": 201, "echo_statement__": 202, "empty_statement": 203, "throw_expression": 204, "var_declaration__": 205, "__var_declaration": 206, "compound_statement": 207, "__inc_statement": 208, "inc_statement__": 209, "__labeled_statement": 210, "labeled_statement__": 211, "jsx_expression": 212, "echo_statement": 213, "send_statement": 214, "__type_conversion_expression": 215, "type_conversion_expression__": 216, "clone_expression": 217, "unset_statement": 218, "method_declaration": 219, "yield_expression__": 220, "__yield_expression": 221, "dec_statement": 222, "__jsx_expression": 223, "jsx_expression__": 224, "clone_expression__": 225, "__clone_expression": 226, "const_declaration__": 227, "__const_declaration": 228, "until_statement": 229, "__require_once_expression": 230, "require_once_expression__": 231, "require_once_expression": 232, "const_declaration": 233, "field_declaration__": 234, "__field_declaration": 235, "include_expression__": 236, "__include_expression": 237, "__dec_statement": 238, "dec_statement__": 239, "__until_statement": 240, "until_statement__": 241, "goto_statement": 242, "for_statement": 243, "type_declaration": 244, "__require_expression": 245, "require_expression__": 246, "nonlocal_statement": 247, "yield_expression": 248, "fallthrough_statement": 249, "include_expression": 250, "require_expression": 251, "class_declaration__": 252, "__class_declaration": 253, "instanceof_expression__": 254, "__instanceof_expression": 255, "type_declaration__": 256, "__type_declaration": 257, "rest_assignment_statement": 258, "variadic_parameter_declaration": 259, "include_once_expression__": 260, "__include_once_expression": 261, "sequence_expression": 262, "include_once_expression": 263, "default_statement": 264, "named_label_statement": 265, "__exponentiation_expression": 266, "exponentiation_expression__": 267, "exponentiation_expression": 268, "__constructor_declaration": 269, "await_expression": 270, "constructor_declaration__": 271, "destructured_left_assignment_statement": 272, "shell_command_expression": 273, "enhanced_for_statement": 274, "__class_constant_access_expression": 275, "debugger_statement": 276, "__parameter_declaration": 277, "parameter_declaration__": 278, "class_constant_access_expression__": 279, "declare_statement": 280, "constructor_declaration": 281, "for_in_statement": 282, "__continue_statement": 283, "try_with_resources_statement": 284, "continue_statement__": 285, "select_statement": 286, "try_statement": 287, "labeled_statement": 288, "expression_switch_statement": 289, "augmented_assignment__": 290, "scoped_property_access_expression__": 291, "yield_statement__": 292, "__yield_statement": 293, "__scoped_property_access_expression": 294, "__augmented_assignment": 295, "with_statement": 296 }, "unk_token": "[UNK]" } }