add generation config and update Readme

README.md

@@ -46,8 +46,8 @@ STEP3-VL-10B delivers best-in-class performance across major multimodal benchmar
 
 | Benchmark         | STEP3-VL-10B (SeRe) | STEP3-VL-10B (PaCoRe) | GLM-4.6V (106B-A12B) | Qwen3-VL (235B-A22B) | Gemini-2.5-Pro | Seed-1.5-VL |
 | :---------------- | :-----------------: | :-------------------: | :------------------: | :------------------: | :------------: | :---------: |
-| **MMMU** | 78.11 | 80.11 | 75.20 | 78.70 | **83.89** | 79.11 |
-| **MathVista** | 83.97 | 85.50 | 83.51 | 85.10 | 83.88 | **85.60** |
+| **MMMU**          |        78.11        |         80.11         |        75.20         |        78.70         |   **83.89**    |    79.11    |
+| **MathVista**     |        83.97        |         85.50         |        83.51         |        85.10         |     83.88      |  **85.60**  |
 | **MathVision**    |        70.81        |       **75.95**       |        63.50         |        72.10         |     73.30      |    68.70    |
 | **MMBench (EN)**  |        92.05        |         92.38         |        92.75         |        92.70         |   **93.19**    |    92.11    |
 | **MMStar**        |        77.48        |         77.64         |        75.30         |        76.80         |   **79.18**    |    77.91    |
@@ -123,7 +123,8 @@ We introduce how to use our model at inference stage using transformers library.
 
 ```python
 from transformers import AutoProcessor, AutoModelForCausalLM
-
+from PIL import Image
+import requests
 
 key_mapping = {
     "^vision_model": "model.vision_model",
@@ -134,16 +135,9 @@ key_mapping = {
 model_path = "stepfun-ai/Step3-VL-10B-Base"
 
 processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
-
-messages = [
-    {
-        "role": "user",
-        "content": [
-            {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg"},
-            {"type": "text", "text": "What's in this picture?"}
-        ]
-    },
-]
+prompt = "<im_patch> What's the content of the image?"
+url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg"
+image = Image.open(requests.get(url, stream=True).raw)
 
 model = AutoModelForCausalLM.from_pretrained(
     model_path,
@@ -152,12 +146,7 @@ model = AutoModelForCausalLM.from_pretrained(
     torch_dtype="auto",
     key_mapping=key_mapping).eval()
 
-
-inputs = processor.apply_chat_template(
-    messages, add_generation_prompt=True, tokenize=True,
-    return_dict=True, return_tensors="pt"
-).to(model.device)
-
+inputs = processor(images=image, text=prompt, return_tensors="pt").to(model.device)
 
 generate_ids = model.generate(**inputs, max_new_tokens=1024, do_sample=False)
 decoded = processor.decode(generate_ids[0, inputs["input_ids"].shape[-1] :], skip_special_tokens=True)
@@ -165,22 +154,24 @@ decoded = processor.decode(generate_ids[0, inputs["input_ids"].shape[-1] :], ski
 print(decoded)
 ```
 
+**Note** It is not recommended to deploy the base model with vLLM; using the Instruct or Chat version is preferred.
+
 ## 📜 Citation
 
 If you find this project useful in your research, please cite our technical report:
 
 ```tex
 @misc{huang2026step3vl10btechnicalreport,
-      title={STEP3-VL-10B Technical Report}, 
+      title={STEP3-VL-10B Technical Report},
       author={Ailin Huang and Chengyuan Yao and Chunrui Han and Fanqi Wan and Hangyu Guo and Haoran Lv and Hongyu Zhou and Jia Wang and Jian Zhou and Jianjian Sun and Jingcheng Hu and Kangheng Lin and Liang Zhao and Mitt Huang and Song Yuan and Wenwen Qu and Xiangfeng Wang and Yanlin Lai and Yingxiu Zhao and Yinmin Zhang and Yukang Shi and Yuyang Chen and Zejia Weng and Ziyang Meng and Ang Li and Aobo Kong and Bo Dong and Changyi Wan and David Wang and Di Qi and Dingming Li and En Yu and Guopeng Li and Haiquan Yin and Han Zhou and Hanshan Zhang and Haolong Yan and Hebin Zhou and Hongbo Peng and Jiaran Zhang and Jiashu Lv and Jiayi Fu and Jie Cheng and Jie Zhou and Jisheng Yin and Jingjing Xie and Jingwei Wu and Jun Zhang and Junfeng Liu and Kaijun Tan and Kaiwen Yan and Liangyu Chen and Lina Chen and Mingliang Li and Qian Zhao and Quan Sun and Shaoliang Pang and Shengjie Fan and Shijie Shang and Siyuan Zhang and Tianhao You and Wei Ji and Wuxun Xie and Xiaobo Yang and Xiaojie Hou and Xiaoran Jiao and Xiaoxiao Ren and Xiangwen Kong and Xin Huang and Xin Wu and Xing Chen and Xinran Wang and Xuelin Zhang and Yana Wei and Yang Li and Yanming Xu and Yeqing Shen and Yuang Peng and Yue Peng and Yu Zhou and Yusheng Li and Yuxiang Yang and Yuyang Zhang and Zhe Xie and Zhewei Huang and Zhenyi Lu and Zhimin Fan and Zihui Cheng and Daxin Jiang and Qi Han and Xiangyu Zhang and Yibo Zhu and Zheng Ge},
       year={2026},
       eprint={2601.09668},
       archivePrefix={arXiv},
       primaryClass={cs.CV},
-      url={https://arxiv.org/abs/2601.09668}, 
+      url={https://arxiv.org/abs/2601.09668},
 }
 ```
 
 ## 📄 License
 
-This project is open-sourced under the [Apache 2.0 License](https://www.google.com/search?q=LICENSE).
+This project is open-sourced under the [Apache 2.0 License](https://www.google.com/search?q=LICENSE).

chat_template.jinja

@@ -1,81 +0,0 @@
-{% macro render_content(content) %}{% if content is none %}{{- '' }}{% elif content is string %}{{- content }}{% elif content is mapping %}{{- content['value'] if 'value' in content else content['text'] }}{% elif content is iterable %}{% for item in content %}{% if item.type == 'text' %}{{- item['value'] if 'value' in item else item['text'] }}{% elif item.type == 'image' %}<im_patch>{% endif %}{% endfor %}{% endif %}{% endmacro %}
-{%- if tools %}
-    {{- '<|im_start|>system\n' }}
-    {%- if messages[0].role == 'system' %}
-        {{- render_content(messages[0].content) + '\n\n' }}
-    {%- endif %}
-    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
-    {%- for tool in tools %}
-        {{- "\n" }}
-        {{- tool | tojson }}
-    {%- endfor %}
-    {{- "\n</tools>\n\nAlways adhere to this exact format for tool use:\n<tool_calls>\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call>\n{additional_tool_calls}</tool_calls>\n\nNote:\n- For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags.\n- `<function-name>` must be an exact match to one of the available tools.\n- `<args-json-object>` must be valid JSON that strictly follows the tool's parameters schema.<|im_end|>\n" }}
-{%- else %}
-    {%- if messages[0].role == 'system' %}
-        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
-    {%- endif %}
-{%- endif %}
-{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
-{%- for message in messages[::-1] %}
-    {%- set index = (messages|length - 1) - loop.index0 %}
-    {%- if ns.multi_step_tool and message.role == "user" and render_content(message.content) is string and not(render_content(message.content).startswith('<tool_response>') and render_content(message.content).endswith('</tool_response>')) %}
-        {%- set ns.multi_step_tool = false %}
-        {%- set ns.last_query_index = index %}
-    {%- endif %}
-{%- endfor %}
-{%- for message in messages %}
-    {%- set content = render_content(message.content) %}
-    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
-        {%- set role_name = 'observation' if (message.role == "system" and not loop.first and message.name == 'observation') else message.role %}
-        {{- '<|im_start|>' + role_name + '\n' + content + '<|im_end|>' + '\n' }}
-    {%- elif message.role == "assistant" %}
-        {%- if message.reasoning_content is string %}
-            {%- set reasoning_content = render_content(message.reasoning_content) %}
-        {%- else %}
-            {%- if '</think>' in content %}
-                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
-                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
-            {%- else %}
-                {%- set reasoning_content = '' %}
-            {%- endif %}
-        {%- endif %}
-        {%- if loop.index0 > ns.last_query_index %}
-            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n' + content }}
-        {%- else %}
-            {{- '<|im_start|>' + message.role + '\n' + content }}
-        {%- endif %}
-        {%- if message.tool_calls %}
-            {{- '\n<tool_calls>' }}
-            {%- for tool_call in message.tool_calls %}
-                {{- '\n' }}
-                {%- if tool_call.function %}
-                    {%- set tool_call = tool_call.function %}
-                {%- endif %}
-                {{- '<tool_call>\n{"name": "' }}
-                {{- tool_call.name }}
-                {{- '", "arguments": ' }}
-                {%- if tool_call.arguments is string %}
-                    {{- tool_call.arguments }}
-                {%- else %}
-                    {{- tool_call.arguments | tojson }}
-                {%- endif %}
-                {{- '}\n</tool_call>' }}
-            {%- endfor %}
-            {{- '\n</tool_calls>' }}
-        {%- endif %}
-        {{- '<|im_end|>\n' }}
-    {%- elif message.role == "tool" %}
-        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
-            {{- '<|im_start|>tool_response' }}
-        {%- endif %}
-        {{- '\n<tool_response>\n' }}
-        {{- content }}
-        {{- '\n</tool_response>' }}
-        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
-            {{- '<|im_end|>\n' }}
-        {%- endif %}
-    {%- endif %}
-{%- endfor %}
-{%- if add_generation_prompt %}
-    {{- '<|im_start|>assistant\n<think>\n' }}
-{%- endif %}

generation_config.json

@@ -0,0 +1,10 @@
+{
+    "temperature": 1.0,
+    "top_p": 1.0,
+    "top_k": 0,
+    "eos_token_id": [
+        151643,
+        151645,
+        151679
+    ]
+}