Upload tokenizer
7f6bcfa verified | | { |
| | "bos_token": { |
| | "content": "<", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | }, |
| | "eos_token": { |
| | "content": ">", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | }, |
| | "mask_token": { |
| | "content": "*", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | }, |
| | "pad_token": { |
| | "content": "-", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | }, |
| | "sep_token": { |
| | "content": "|", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | }, |
| | "unk_token": { |
| | "content": "X", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | } |
| | } |
| |
|