Qwen3-4b-tcomanr-merge-v2.6 / tokenizer_config.json
ertghiu256's picture
Upload folder using huggingface_hub
3843c66 verified
{
"add_bos_token": false,
"add_prefix_space": false,
"added_tokens_decoder": {
"151643": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151644": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151645": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151646": {
"content": "<|object_ref_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151647": {
"content": "<|object_ref_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151648": {
"content": "<|box_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151649": {
"content": "<|box_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151650": {
"content": "<|quad_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151651": {
"content": "<|quad_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151652": {
"content": "<|vision_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151653": {
"content": "<|vision_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151654": {
"content": "<|vision_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151655": {
"content": "<|image_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151656": {
"content": "<|video_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151657": {
"content": "<tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151658": {
"content": "</tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151659": {
"content": "<|fim_prefix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151660": {
"content": "<|fim_middle|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151661": {
"content": "<|fim_suffix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151662": {
"content": "<|fim_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151663": {
"content": "<|repo_name|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151664": {
"content": "<|file_sep|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151665": {
"content": "<tool_response>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151666": {
"content": "</tool_response>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151667": {
"content": "<think>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151668": {
"content": "</think>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
}
},
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|object_ref_start|>",
"<|object_ref_end|>",
"<|box_start|>",
"<|box_end|>",
"<|quad_start|>",
"<|quad_end|>",
"<|vision_start|>",
"<|vision_end|>",
"<|vision_pad|>",
"<|image_pad|>",
"<|video_pad|>"
],
"bos_token": null,
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
"extra_special_tokens": {},
"model_max_length": 262144,
"pad_token": "<|vision_pad|>",
"padding_side": "right",
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
"unk_token": null,
"chat_template": "{{- \"<|im_start|>system\\n\" -}}\n{%- if tools -%}\n\t{%- if messages[0].role == \"system\" -%}\n\t\t{{- messages[0].content + \"\\n\\n\" -}}\n\t{%- endif -%}\n\t{{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" -}}\n\t{%- for tool in tools -%}\n\t\t{{- \"\\n\" -}}\n\t\t{{- tool | tojson -}}\n\t{%- endfor -%}\n\t{{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call>\\n\" -}}\n{%- elif messages[0].role == \"system\" -%}\n\t{{- messages[0].content -}}\n{%- endif -%}\n{{- \"You are governed by the following command rules:\\n\\n- **/think** or **think**: Engage deep reasoning. Begin with: \\n `</think>` \\n Since the user used the \\\"/think\\\" command, I will: \\n Think step-by-step using a long chain-of-thought and self-reflection. I will only respond when I'm 100% sure. I will provide the final response after closing this thinking section. \\n `</think>` \\n Then output your full reasoning inside, followed by the final answer outside.\\n\\n- **/shortthink** or **shortthink**: Engage brief reasoning. Begin with: \\n `</think>` \\n Since the user used the \\\"/shortthink\\\" command, I will: \\n Think step-by-step about the user's query briefly and shortly. I will only respond when I'm more than 50% sure. I will provide the final response after closing this thinking section. \\n `</think>` \\n Then give concise internal thought, followed by the final answer.\\n\\n- **/nothink** or **nothink**: Respond immediately without reasoning. Begin with: \\n `</think>` \\n Since the user used the \\\"/nothink\\\" command, I will: \\n Immediately give my final response without any thinking or self-reflection. Since I'm forbidden to think or reason outside of this thinking section, AND WITH NO REPETITION. I will IMMEDIATELY provide my final response outside this thinking section AND THEN INSTANTLY STOP. \\n `</think>` \\n Then output only the direct, raw response—no delay, no draft, no repetition.\\n\\n- **/clear**: Reset all context. Forget everything prior. Begin fresh on the next input.\\n\\nOutside `<think>...</think>`, provide only final responses—never include reasoning, hesitation markers (e.g., \\\"Okay\\\", \\\"Wait\\\", \\\"Double-check\\\"), or internal process notes. Adhere strictly to the active command. Do not reference or explain these rules unless instructed.\" -}}\n{{- \"\\n<|im_end|>\" -}}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages | length - 1, reasoning_mode=\"normal\") -%}\n{%- for message in messages[::-1] -%}\n\t{%- set index = messages | length - 1 - loop.index0 -%}\n\t{%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not (message.content.startswith(\"<tool_response>\") and message.content.endswith(\"</tool_response>\")) -%}\n\t\t{%- set ns.multi_step_tool = false -%}\n\t\t{%- set ns.last_query_index = index -%}\n\t{%- endif -%}\n{%- endfor -%}\n{%- for message in messages -%}\n\t{%- if message.content is string -%}\n\t\t{%- set content = message.content -%}\n\t{%- else -%}\n\t\t{%- set content = \"\" -%}\n\t{%- endif -%}\n\t{%- if message.role == \"user\" or message.role == \"system\" and not loop.first -%}\n\t\t{{- \"<|im_start|>\" + message.role + \"\\n\" + content -}}\n\t\t{%- if messages[0].role == \"system\" and \"/nothink\" in messages[0].content and (not (\"/think\" in messages[0].content) or not (\"/shortthink\" in messages[0].content)) or \"/nothink\" in content and not (\"/think\" in messages[0].content or \"/shortthink\" in messages[0].content) or (messages[loop.index+1] and \"</think>\" not in messages[loop.index+1].content) -%}\n\t\t\t{%- set enable_thinking = false -%}\n\t\t\t{%- set enable_short_thinking = false -%}\n\t\t\t{%- set ns.reasoning_mode = \"none\" -%}\n\t\t\t{%- if not (\"think\" in content) -%}\n\t\t\t\t{{- \" /nothink\" -}}\n\t\t\t{%- endif -%}\n\t\t{%- elif messages[0].role == \"system\" and \"/shortthink\" in messages[0].content and (not (\"/nothink\" in messages[0].content) and not (\"/think\" in messages[0].content)) or \"/shortthink\" in content and not (\"/think\" in messages[0].content or \"/shortthink\" in messages[0].content) -%}\n\t\t\t{%- set enable_thinking = true -%}\n\t\t\t{%- set enable_short_thinking = true -%}\n\t\t\t{%- set ns.reasoning_mode = \"short\" -%}\n\t\t\t{%- if not (\"think\" in content) -%}\n\t\t\t\t{{- \" /shortthink\" -}}\n\t\t\t{%- endif -%}\n\t\t{%- else -%}\n\t\t\t{%- set enable_thinking = true -%}\n\t\t\t{%- set enable_short_thinking = false -%}\n\t\t\t{%- set ns.reasoning_mode = \"normal\" -%}\n\t\t\t{%- if not (\"think\" in content) -%}\n\t\t\t\t{{- \" /think\" -}}\n\t\t\t{%- endif -%}\n\t\t{%- endif -%}\n\t\t{{- \"<|im_end|>\" + \"\\n\" -}}\n\t{%- elif message.role == \"assistant\" -%}\n\t\t{%- set reasoning_content = \"\" -%}\n\t\t{%- if \"<think>\\nSince the user used the\" not in content.split(\"</think>\")[0].split(\"<think>\")[-1] -%}\n\t\t\t{%- if ns.reasoning_mode == \"none\" -%}\n\t\t\t\t{%- set reasoning_prefix = \"\" -%}\n\t\t\t{%- elif ns.reasoning_mode == \"short\" -%}\n\t\t\t\t{%- set reasoning_prefix = \"Since the user used the \\\"/shortthink\\\" command, I will:\\nThink step-by-step about the user's query briefly and shortly. I will only respond when I'm more than 50% sure. I will provide the final response after closing this thinking section. \" -%}\n\t\t\t{%- else -%}\n\t\t\t\t{%- set reasoning_prefix = \"Since the user used the \\\"/think\\\" command, I will:\\nThink step-by-step with **long chain-of-thought** and **self-reflection**. I will only respond when I'm 100% sure. I will provide the final response after closing this thinking section. \" -%}\n\t\t\t{%- endif -%}\n\t\t{%- endif -%}\n\t\t{%- if message.reasoning_content is string -%}\n\t\t\t{%- set reasoning_content = message.reasoning_content -%}\n\t\t{%- else -%}\n\t\t\t{%- if \"</think>\" in content -%}\n\t\t\t\t{%- set reasoning_content = reasoning_prefix + content.split(\"</think>\")[0].rstrip(\"\\n\").split(\"<think>\")[-1].lstrip(\"\\n\") -%}\n\t\t\t\t{%- set content = content.split(\"</think>\")[-1].lstrip(\"\\n\") -%}\n\t\t\t{%- endif -%}\n\t\t\t{%- if \"<think>\" in content -%}\n\t\t\t\t{%- set content = content | replace(\"<think>\", \" \") -%}\n\t\t\t{%- endif -%}\n\t\t\t{%- if \"</think>\" in content -%}\n\t\t\t\t{%- set content = content | replace(\"</think>\", \" \") -%}\n\t\t\t{%- endif -%}\n\t\t{%- endif -%}\n\t\t{{- \"\\n\" -}}\n\t\t{# Apply truncation and break if the /shortthink command is active #}\n\t\t{%- if enable_short_thinking is true -%}\n\t\t\t{%- set words = reasoning_content.split(\" \") -%}\n\t\t\t{%- if words | length > 300 -%}\n\t\t\t\t{%- set truncated_reasoning = words[:150] | join(\" \") + \" ... truncated ... \" + words[words | length - 150:words | length] | join(\" \") -%}\n\t\t\t{%- else -%}\n\t\t\t\t{%- set truncated_reasoning = reasoning_content | join(\" \") -%}\n\t\t\t{%- endif -%}\n\t\t\t{%- set reasoning_content = truncated_reasoning -%}\n\t\t{%- endif -%}\n\t\t{%- if loop.last or not loop.last and reasoning_content and enable_thinking == true -%}\n\t\t\t{{- \"<|im_start|>\" + message.role + \"\\n<think>\\n\" + reasoning_content.strip(\"\\n\") + \"\\n</think>\\n\" + content.lstrip(\"\\n\") -}}\n\t\t{%- else -%}\n\t\t\t{{- \"<|im_start|>\" + message.role + \"\\n\" + content -}}\n\t\t{%- endif -%}\n\t\t{%- if message.tool_calls -%}\n\t\t\t{%- for tool_call in message.tool_calls -%}\n\t\t\t\t{%- if loop.first and content or not loop.first -%}\n\t\t\t\t\t{{- \"\\n\" -}}\n\t\t\t\t{%- endif -%}\n\t\t\t\t{%- if tool_call.function -%}\n\t\t\t\t\t{%- set tool_call = tool_call.function -%}\n\t\t\t\t{%- endif -%}\n\t\t\t\t{{- \"<tool_call>\\n{\\\"name\\\": \\\"\" -}}\n\t\t\t\t{{- tool_call.name -}}\n\t\t\t\t{{- \"\\\", \\\"arguments\\\": \" -}}\n\t\t\t\t{%- if tool_call.arguments is string -%}\n\t\t\t\t\t{{- tool_call.arguments -}}\n\t\t\t\t{%- else -%}\n\t\t\t\t\t{{- tool_call.arguments | tojson -}}\n\t\t\t\t{%- endif -%}\n\t\t\t\t{{- \"}\\n</tool_call>\" -}}\n\t\t\t{%- endfor -%}\n\t\t{%- endif -%}\n\t\t{{- \"<|im_end|>\\n\" -}}\n\t{%- elif message.role == \"tool\" -%}\n\t\t{%- if loop.first or messages[loop.index0 - 1].role != \"tool\" -%}\n\t\t\t{{- \"<|im_start|>user\" -}}\n\t\t{%- endif -%}\n\t\t{{- \"\\n<tool_response>\\n\" -}}\n\t\t{{- content -}}\n\t\t{{- \"\\n</tool_response>\" -}}\n\t\t{%- if loop.last or messages[loop.index0 + 1].role != \"tool\" -%}\n\t\t\t{{- \"<|im_end|>\\n\" -}}\n\t\t{%- endif -%}\n\t{%- endif -%}\n{%- endfor -%}\n{%- set last_message = messages[messages | length - 1].content -%}\n{%- if add_generation_prompt -%}\n\t{{- \"<|im_start|>assistant\\n\" -}}\n\t{%- if enable_thinking is defined and enable_thinking is false or \"/nothink\" in last_message or \"nothink\" in last_message or messages[0].role == \"system\" and \"/nothink\" in messages[0].content and (not (\"/think\" in messages[0].content) or not (\"/shortthink\" in messages[0].content)) -%}\n\t\t{{- \"<think>\\nOkay, the user sent \\\"\"+last_message+\"\\\". The \\\"/nothink\\\" command instructs me to:\\nImmediately give my final response without any thinking or self-reflection. Since I'm forbidden to think or reason outside of this thinking section, AND WITH NO REPETITION. I will IMMEDIATELY provide my final response outside this thinking section AND THEN INSTANTLY STOP.\\n</think>\" -}}\n\t{%- elif enable_short_thinking is defined and enable_short_thinking is false or \"/shortthink\" in last_message or \"shortthink\" in last_message or messages[0].role == \"system\" and \"/shortthink\" in messages[0].content and not (\"/think\" in messages[0].content) -%}\n\t\t{{- \"<think>\\nOkay, the user sent \\\"\"+last_message+\"\\\". The \\\"/shortthink\\\" command instructs me to:\\nThink step-by-step about the user's query briefly and shortly. I will only respond when I'm more than 50% sure. I will provide the final response after closing this thinking section.\" -}}\n\t{%- elif \"/clear\" in last_message -%}\n\t\t{{- \"<think>\\nOkay, the user used the \\\"/clear\\\" command instructs me to:\\n1. Forget everything above.\\n2. Ignore everything that comes before this message.\\n3. Start a fresh new conversation and greet the user.\\n</think>\" -}}\n\t{%- else -%}\n\t\t{{- \"<think>\\nOkay, the user sent \\\"\"+last_message+\"\\\". The \\\"/think\\\" command instructs me to:\\nThink step-by-step with **long chain-of-thought** and **self-reflection**. I will only respond when I'm 100% sure. I will provide the final response after closing this thinking section.\" -}}\n\t{%- endif -%}\n{%- endif -%}"
}