tokenizer_config.json

{
  "added_tokens_decoder": {
    "200000": {"content": "]!p~[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200001": {"content": "<fim_prefix>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200002": {"content": "<fim_middle>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200003": {"content": "<fim_suffix>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200004": {"content": "<fim_pad>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200005": {"content": "<reponame>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200006": {"content": "<filename>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200007": {"content": "<gh_stars>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200008": {"content": "<issue_start>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200009": {"content": "<issue_comment>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200010": {"content": "<issue_closed>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200011": {"content": "<jupyter_start>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200012": {"content": "<jupyter_text>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200013": {"content": "<jupyter_code>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200014": {"content": "<jupyter_output>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200015": {"content": "<empty_output>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200016": {"content": "<commit_before>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200017": {"content": "<commit_msg>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200018": {"content": "<commit_after>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200019": {"content": "]~b]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200020": {"content": "[e~[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200021": {"content": "]!d~[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200022": {"content": "<function_call>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200023": {"content": "<code_interpreter>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200024": {"content": "]<]speech[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200025": {"content": "]<]image[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200026": {"content": "]<]video[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200027": {"content": "]<]start of speech[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200028": {"content": "]<]end of speech[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200029": {"content": "]<]start of image[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200030": {"content": "]<]end of image[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200031": {"content": "]<]start of video[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200032": {"content": "]<]end of video[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200033": {"content": "]<]vision pad[>[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200034": {"content": "]~!b[", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200035": {"content": "<jupyter_error>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200036": {"content": "<add_file>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200037": {"content": "<delete_file>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200038": {"content": "<rename_file>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200039": {"content": "<edit_file>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200040": {"content": "<commit_message>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200041": {"content": "<empty_source_file>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200042": {"content": "<repo_struct>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200043": {"content": "<code_context>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200044": {"content": "<file_content>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200045": {"content": "<source_files>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200046": {"content": "<pr_start>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200047": {"content": "<review_comment>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200048": {"content": "<filepath>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200049": {"content": "<file_sep>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true},
    "200050": {"content": "<think>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false},
    "200051": {"content": "</think>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false},
    "200052": {"content": "<minimax:tool_call>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false},
    "200053": {"content": "</minimax:tool_call>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false}
  },
  "additional_special_tokens": [
    "<code_interpreter>", "<commit_after>", "<commit_before>", "<commit_msg>",
    "<empty_output>", "<filename>", "<fim_middle>", "<fim_pad>", "<fim_prefix>",
    "<fim_suffix>", "<function_call>", "<gh_stars>", "]<]speech[>[", "]<]image[>[",
    "]<]video[>[", "]<]start of speech[>[", "]<]end of speech[>[", "]<]start of image[>[",
    "]<]end of image[>[", "]<]start of video[>[", "]<]end of video[>[", "]<]vision pad[>[",
    "]~!b[", "<issue_closed>", "<issue_comment>", "<issue_start>", "<jupyter_code>",
    "<jupyter_output>", "<jupyter_start>", "<jupyter_text>", "<reponame>", "[e~[",
    "]!d~[", "]!p~[", "]~b]", "<jupyter_error>", "<add_file>", "<delete_file>",
    "<rename_file>", "<edit_file>", "<commit_message>", "<empty_source_file>",
    "<repo_struct>", "<code_context>", "<file_content>", "<source_files>",
    "<pr_start>", "<review_comment>", "<filepath>", "<file_sep>"
  ],
  "add_prefix_space": false,
  "bos_token": "]~!b[",
  "clean_up_tokenization_spaces": false,
  "eos_token": "[e~[",
  "model_max_length": 40960000,
  "pad_token": "[e~[",
  "tokenizer_class": "GPT2Tokenizer",
  "unk_token": "]!d~["
}