tokyotech-llm
/

Swallow-13b-NVE-hf

Text Generation

text-generation-inference

Model card Files Files and versions

Taishi-N324 commited on Feb 3, 2024

Commit

d1042b5

·

verified ·

1 Parent(s): 29624c3

Upload tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json +3 -3

tokenizer_config.json CHANGED Viewed

@@ -5,7 +5,7 @@
     "__type": "AddedToken",
     "content": "<s>",
     "lstrip": false,
-    "normalized": false,
     "rstrip": false,
     "single_word": false
   },
@@ -14,7 +14,7 @@
     "__type": "AddedToken",
     "content": "</s>",
     "lstrip": false,
-    "normalized": false,
     "rstrip": false,
     "single_word": false
   },
@@ -28,7 +28,7 @@
     "__type": "AddedToken",
     "content": "<unk>",
     "lstrip": false,
-    "normalized": false,
     "rstrip": false,
     "single_word": false
   }

     "__type": "AddedToken",
     "content": "<s>",
     "lstrip": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   },
     "__type": "AddedToken",
     "content": "</s>",
     "lstrip": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   },
     "__type": "AddedToken",
     "content": "<unk>",
     "lstrip": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   }