Set tokenizer "model_max_length" property to 16384

#1
by yjoonjang - opened
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -312,7 +312,7 @@
312
  "eos_token": "<\\s>",
313
  "extra_special_tokens": {},
314
  "mask_token": "<mask>",
315
- "model_max_length": 1024,
316
  "pad_token": "<pad>",
317
  "sep_token": "<sep>",
318
  "strip_accents": null,
 
312
  "eos_token": "<\\s>",
313
  "extra_special_tokens": {},
314
  "mask_token": "<mask>",
315
+ "model_max_length": 16384,
316
  "pad_token": "<pad>",
317
  "sep_token": "<sep>",
318
  "strip_accents": null,