jaeyong2
/

Qwen2.5-1.5B-Instruct-Ja-SFT

Text Generation

text-generation-inference

Model card Files Files and versions

jaeyong2 commited on Oct 15, 2024

Commit

6e715fb

·

verified ·

1 Parent(s): 0388df8

Upload tokenizer

Files changed (2) hide show

tokenizer.json +2 -2
tokenizer_config.json +1 -1

tokenizer.json CHANGED Viewed

@@ -2,13 +2,13 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 2048,
     "strategy": "LongestFirst",
     "stride": 0
   },
   "padding": {
     "strategy": {
-      "Fixed": 2048
     },
     "direction": "Right",
     "pad_to_multiple_of": null,

   "version": "1.0",
   "truncation": {
     "direction": "Right",
+    "max_length": 4096,
     "strategy": "LongestFirst",
     "stride": 0
   },
   "padding": {
     "strategy": {
+      "Fixed": 4096
     },
     "direction": "Right",
     "pad_to_multiple_of": null,

tokenizer_config.json CHANGED Viewed

@@ -199,7 +199,7 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",
-  "max_length": 2048,
   "model_max_length": 131072,
   "pad_to_multiple_of": null,
   "pad_token": "<|endoftext|>",

   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",
+  "max_length": 4096,
   "model_max_length": 131072,
   "pad_to_multiple_of": null,
   "pad_token": "<|endoftext|>",