hsuvaskakoty commited on
Commit
2cc888e
·
1 Parent(s): 1e9f4e8

Upload 9 files

Browse files
added_tokens.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "</t>": 50268,
3
+ "</y>": 50266,
4
+ "<t>": 50267,
5
+ "<y>": 50265
6
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
pytorch_model.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fc8b5e3df98c686cc4b229b8e809bb4fe5b23fefa2d0fcae0a83fc70d9d5a950
3
  size 498677429
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e0a534ef949cb38e587d890ee909b7eb3fb2694f70c0d52e9399649f0c12829
3
  size 498677429
special_tokens_map.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<y>",
4
+ "</y>",
5
+ "<t>",
6
+ "</t>"
7
+ ],
8
+ "bos_token": "<s>",
9
+ "cls_token": "<s>",
10
+ "eos_token": "</s>",
11
+ "mask_token": {
12
+ "content": "<mask>",
13
+ "lstrip": true,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false
17
+ },
18
+ "pad_token": "<pad>",
19
+ "sep_token": "</s>",
20
+ "unk_token": "<unk>"
21
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": "<s>",
4
+ "clean_up_tokenization_spaces": true,
5
+ "cls_token": "<s>",
6
+ "eos_token": "</s>",
7
+ "errors": "replace",
8
+ "mask_token": "<mask>",
9
+ "model_max_length": 512,
10
+ "pad_token": "<pad>",
11
+ "sep_token": "</s>",
12
+ "tokenizer_class": "RobertaTokenizer",
13
+ "trim_offsets": true,
14
+ "unk_token": "<unk>"
15
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff