| { | |
| "version": "1.0", | |
| "truncation": null, | |
| "padding": null, | |
| "added_tokens": [ | |
| { | |
| "id": 0, | |
| "content": "[PAD]", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 1, | |
| "content": "[SOS]", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 2, | |
| "content": "[EOS]", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 3, | |
| "content": "[UNK]", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 4, | |
| "content": "[MSK]", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 5, | |
| "content": "[SEP]", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| } | |
| ], | |
| "normalizer": { | |
| "type": "Sequence", | |
| "normalizers": [ | |
| { | |
| "type": "NFD" | |
| }, | |
| { | |
| "type": "StripAccents" | |
| }, | |
| { | |
| "type": "Strip", | |
| "strip_left": true, | |
| "strip_right": true | |
| } | |
| ] | |
| }, | |
| "pre_tokenizer": { | |
| "type": "Whitespace" | |
| }, | |
| "post_processor": null, | |
| "decoder": null, | |
| "model": { | |
| "type": "WordLevel", | |
| "vocab": { | |
| "[PAD]": 0, | |
| "[SOS]": 1, | |
| "[EOS]": 2, | |
| "[UNK]": 3, | |
| "[MSK]": 4, | |
| "[SEP]": 5, | |
| "expression_statement__": 6, | |
| "__expression_statement": 7, | |
| "assignment": 8, | |
| "__if_statement": 9, | |
| "if_statement__": 10, | |
| "expression_statement": 11, | |
| "return_statement": 12, | |
| "__parenthesized_expression": 13, | |
| "parenthesized_expression__": 14, | |
| "binary_expression": 15, | |
| "binary_expression__": 16, | |
| "__binary_expression": 17, | |
| "__call_expression": 18, | |
| "call_expression__": 19, | |
| "assignment_expression__": 20, | |
| "__assignment_expression": 21, | |
| "member_expression": 22, | |
| "selector_expression": 23, | |
| "compound_statement__": 24, | |
| "__compound_statement": 25, | |
| "local_variable_declaration": 26, | |
| "return_statement__": 27, | |
| "__return_statement": 28, | |
| "for_statement__": 29, | |
| "__for_statement": 30, | |
| "assignment_expression": 31, | |
| "member_access_expression": 32, | |
| "object_creation_expression": 33, | |
| "call_statement": 34, | |
| "member_call_expression": 35, | |
| "subscript_expression": 36, | |
| "local_variable_declaration__": 37, | |
| "__local_variable_declaration": 38, | |
| "member_call_expression__": 39, | |
| "__member_call_expression": 40, | |
| "parenthesized_expression": 41, | |
| "__assignment_statement": 42, | |
| "assignment_statement__": 43, | |
| "try_statement__": 44, | |
| "__try_statement": 45, | |
| "call_expression": 46, | |
| "unary_expression": 47, | |
| "function_call_expression": 48, | |
| "raise_statement": 49, | |
| "__member_expression": 50, | |
| "member_expression__": 51, | |
| "function_call_expression__": 52, | |
| "__function_call_expression": 53, | |
| "__short_var_declaration": 54, | |
| "short_var_declaration__": 55, | |
| "subscript_expression__": 56, | |
| "__subscript_expression": 57, | |
| "call_statement__": 58, | |
| "__call_statement": 59, | |
| "array_creation_expression": 60, | |
| "selector_expression__": 61, | |
| "__selector_expression": 62, | |
| "__throw_statement": 63, | |
| "throw_statement__": 64, | |
| "cast_expression": 65, | |
| "update_expression": 66, | |
| "object_creation_expression__": 67, | |
| "__object_creation_expression": 68, | |
| "augmented_assignment": 69, | |
| "unary_expression__": 70, | |
| "__unary_expression": 71, | |
| "break_statement": 72, | |
| "__unary_op_expression": 73, | |
| "unary_op_expression__": 74, | |
| "enhanced_for_statement__": 75, | |
| "__enhanced_for_statement": 76, | |
| "scoped_call_expression": 77, | |
| "class_constant_access_expression": 78, | |
| "assignment_statement": 79, | |
| "while_statement__": 80, | |
| "__while_statement": 81, | |
| "if_statement": 82, | |
| "continue_statement": 83, | |
| "ternary_expression__": 84, | |
| "__ternary_expression": 85, | |
| "__foreach_statement": 86, | |
| "foreach_statement__": 87, | |
| "with_statement__": 88, | |
| "__with_statement": 89, | |
| "array_creation_expression__": 90, | |
| "__array_creation_expression": 91, | |
| "instanceof_expression": 92, | |
| "throw_expression__": 93, | |
| "__throw_expression": 94, | |
| "assert_statement": 95, | |
| "conditional_expression__": 96, | |
| "__conditional_expression": 97, | |
| "__scoped_call_expression": 98, | |
| "scoped_call_expression__": 99, | |
| "index_expression": 100, | |
| "__member_access_expression": 101, | |
| "member_access_expression__": 102, | |
| "cast_expression__": 103, | |
| "__cast_expression": 104, | |
| "index_expression__": 105, | |
| "__index_expression": 106, | |
| "__augmented_assignment_expression": 107, | |
| "augmented_assignment_expression__": 108, | |
| "pass_statement": 109, | |
| "scoped_property_access_expression": 110, | |
| "new_expression": 111, | |
| "import_from_statement": 112, | |
| "unless_statement": 113, | |
| "var_declaration": 114, | |
| "else_statement": 115, | |
| "__else_statement": 116, | |
| "else_statement__": 117, | |
| "method_declaration__": 118, | |
| "__method_declaration": 119, | |
| "new_expression__": 120, | |
| "__new_expression": 121, | |
| "unary_op_expression": 122, | |
| "short_var_declaration": 123, | |
| "__case_statement": 124, | |
| "case_statement__": 125, | |
| "__anonymous_function_creation_expression": 126, | |
| "anonymous_function_creation_expression__": 127, | |
| "augmented_assignment_expression": 128, | |
| "__defer_statement": 129, | |
| "defer_statement__": 130, | |
| "delete_statement": 131, | |
| "throw_statement": 132, | |
| "parameter_declaration": 133, | |
| "import_statement": 134, | |
| "when_statement": 135, | |
| "when_statement__": 136, | |
| "__when_statement": 137, | |
| "__lambda_expression": 138, | |
| "lambda_expression__": 139, | |
| "ternary_expression": 140, | |
| "switch_expression__": 141, | |
| "__switch_expression": 142, | |
| "__synchronized_statement": 143, | |
| "synchronized_statement__": 144, | |
| "print_statement": 145, | |
| "lambda_expression": 146, | |
| "switch_statement__": 147, | |
| "__switch_statement": 148, | |
| "operator_assignment_statement": 149, | |
| "type_assertion_expression__": 150, | |
| "__type_assertion_expression": 151, | |
| "rescue_statement": 152, | |
| "__assert_statement": 153, | |
| "assert_statement__": 154, | |
| "for_in_statement__": 155, | |
| "__for_in_statement": 156, | |
| "__rescue_statement": 157, | |
| "rescue_statement__": 158, | |
| "__unset_statement": 159, | |
| "unset_statement__": 160, | |
| "type_assertion_expression": 161, | |
| "__assignment": 162, | |
| "assignment__": 163, | |
| "__unless_statement": 164, | |
| "unless_statement__": 165, | |
| "expression_switch_statement__": 166, | |
| "__expression_switch_statement": 167, | |
| "inc_statement": 168, | |
| "slice_expression": 169, | |
| "__slice_expression": 170, | |
| "slice_expression__": 171, | |
| "__operator_assignment_statement": 172, | |
| "operator_assignment_statement__": 173, | |
| "__try_with_resources_statement": 174, | |
| "try_with_resources_statement__": 175, | |
| "global_statement": 176, | |
| "receive_statement__": 177, | |
| "__receive_statement": 178, | |
| "field_declaration": 179, | |
| "await_expression__": 180, | |
| "__await_expression": 181, | |
| "type_switch_statement__": 182, | |
| "__type_switch_statement": 183, | |
| "do_statement__": 184, | |
| "__do_statement": 185, | |
| "__go_statement": 186, | |
| "go_statement__": 187, | |
| "case_statement": 188, | |
| "type_conversion_expression": 189, | |
| "sequence_expression__": 190, | |
| "__sequence_expression": 191, | |
| "__update_expression": 192, | |
| "update_expression__": 193, | |
| "default_statement__": 194, | |
| "__default_statement": 195, | |
| "select_statement__": 196, | |
| "__select_statement": 197, | |
| "conditional_expression": 198, | |
| "__send_statement": 199, | |
| "send_statement__": 200, | |
| "__echo_statement": 201, | |
| "echo_statement__": 202, | |
| "empty_statement": 203, | |
| "throw_expression": 204, | |
| "var_declaration__": 205, | |
| "__var_declaration": 206, | |
| "compound_statement": 207, | |
| "__inc_statement": 208, | |
| "inc_statement__": 209, | |
| "__labeled_statement": 210, | |
| "labeled_statement__": 211, | |
| "jsx_expression": 212, | |
| "echo_statement": 213, | |
| "send_statement": 214, | |
| "__type_conversion_expression": 215, | |
| "type_conversion_expression__": 216, | |
| "clone_expression": 217, | |
| "unset_statement": 218, | |
| "method_declaration": 219, | |
| "yield_expression__": 220, | |
| "__yield_expression": 221, | |
| "dec_statement": 222, | |
| "__jsx_expression": 223, | |
| "jsx_expression__": 224, | |
| "clone_expression__": 225, | |
| "__clone_expression": 226, | |
| "const_declaration__": 227, | |
| "__const_declaration": 228, | |
| "until_statement": 229, | |
| "__require_once_expression": 230, | |
| "require_once_expression__": 231, | |
| "require_once_expression": 232, | |
| "const_declaration": 233, | |
| "field_declaration__": 234, | |
| "__field_declaration": 235, | |
| "include_expression__": 236, | |
| "__include_expression": 237, | |
| "__dec_statement": 238, | |
| "dec_statement__": 239, | |
| "__until_statement": 240, | |
| "until_statement__": 241, | |
| "goto_statement": 242, | |
| "for_statement": 243, | |
| "type_declaration": 244, | |
| "__require_expression": 245, | |
| "require_expression__": 246, | |
| "nonlocal_statement": 247, | |
| "yield_expression": 248, | |
| "fallthrough_statement": 249, | |
| "include_expression": 250, | |
| "require_expression": 251, | |
| "class_declaration__": 252, | |
| "__class_declaration": 253, | |
| "instanceof_expression__": 254, | |
| "__instanceof_expression": 255, | |
| "type_declaration__": 256, | |
| "__type_declaration": 257, | |
| "rest_assignment_statement": 258, | |
| "variadic_parameter_declaration": 259, | |
| "include_once_expression__": 260, | |
| "__include_once_expression": 261, | |
| "sequence_expression": 262, | |
| "include_once_expression": 263, | |
| "default_statement": 264, | |
| "named_label_statement": 265, | |
| "__exponentiation_expression": 266, | |
| "exponentiation_expression__": 267, | |
| "exponentiation_expression": 268, | |
| "__constructor_declaration": 269, | |
| "await_expression": 270, | |
| "constructor_declaration__": 271, | |
| "destructured_left_assignment_statement": 272, | |
| "shell_command_expression": 273, | |
| "enhanced_for_statement": 274, | |
| "__class_constant_access_expression": 275, | |
| "debugger_statement": 276, | |
| "__parameter_declaration": 277, | |
| "parameter_declaration__": 278, | |
| "class_constant_access_expression__": 279, | |
| "declare_statement": 280, | |
| "constructor_declaration": 281, | |
| "for_in_statement": 282, | |
| "__continue_statement": 283, | |
| "try_with_resources_statement": 284, | |
| "continue_statement__": 285, | |
| "select_statement": 286, | |
| "try_statement": 287, | |
| "labeled_statement": 288, | |
| "expression_switch_statement": 289, | |
| "augmented_assignment__": 290, | |
| "scoped_property_access_expression__": 291, | |
| "yield_statement__": 292, | |
| "__yield_statement": 293, | |
| "__scoped_property_access_expression": 294, | |
| "__augmented_assignment": 295, | |
| "with_statement": 296 | |
| }, | |
| "unk_token": "[UNK]" | |
| } | |
| } |