pt-tupi-tokenizer / vocab.json
CASLL's picture
Upload tokenizer
c59e1e6 verified
{"<s>":0,"<pad>":1,"</s>":2,"<unk>":3,"<mask>":4,"!":5,"\"":6,"#":7,"$":8,"%":9,"&":10,"'":11,"(":12,")":13,"*":14,"+":15,",":16,"-":17,".":18,"/":19,"0":20,"1":21,"2":22,"3":23,"4":24,"5":25,"6":26,"7":27,"8":28,"9":29,":":30,";":31,"<":32,"=":33,">":34,"?":35,"@":36,"A":37,"B":38,"C":39,"D":40,"E":41,"F":42,"G":43,"H":44,"I":45,"J":46,"K":47,"L":48,"M":49,"N":50,"O":51,"P":52,"Q":53,"R":54,"S":55,"T":56,"U":57,"V":58,"W":59,"X":60,"Y":61,"Z":62,"[":63,"\\":64,"]":65,"^":66,"_":67,"`":68,"a":69,"b":70,"c":71,"d":72,"e":73,"f":74,"g":75,"h":76,"i":77,"j":78,"k":79,"l":80,"m":81,"n":82,"o":83,"p":84,"q":85,"r":86,"s":87,"t":88,"u":89,"v":90,"w":91,"x":92,"y":93,"z":94,"{":95,"|":96,"}":97,"~":98,"¡":99,"¢":100,"£":101,"¤":102,"¥":103,"¦":104,"§":105,"¨":106,"©":107,"ª":108,"«":109,"¬":110,"®":111,"¯":112,"°":113,"±":114,"²":115,"³":116,"´":117,"µ":118,"¶":119,"·":120,"¸":121,"¹":122,"º":123,"»":124,"¼":125,"½":126,"¾":127,"¿":128,"À":129,"Á":130,"Â":131,"Ã":132,"Ä":133,"Å":134,"Æ":135,"Ç":136,"È":137,"É":138,"Ê":139,"Ë":140,"Ì":141,"Í":142,"Î":143,"Ï":144,"Ð":145,"Ñ":146,"Ò":147,"Ó":148,"Ô":149,"Õ":150,"Ö":151,"×":152,"Ø":153,"Ù":154,"Ú":155,"Û":156,"Ü":157,"Ý":158,"Þ":159,"ß":160,"à":161,"á":162,"â":163,"ã":164,"ä":165,"å":166,"æ":167,"ç":168,"è":169,"é":170,"ê":171,"ë":172,"ì":173,"í":174,"î":175,"ï":176,"ð":177,"ñ":178,"ò":179,"ó":180,"ô":181,"õ":182,"ö":183,"÷":184,"ø":185,"ù":186,"ú":187,"û":188,"ü":189,"ý":190,"þ":191,"ÿ":192,"Ā":193,"ā":194,"Ă":195,"ă":196,"Ą":197,"ą":198,"Ć":199,"ć":200,"Ĉ":201,"ĉ":202,"Ċ":203,"ċ":204,"Č":205,"č":206,"Ď":207,"ď":208,"Đ":209,"đ":210,"Ē":211,"ē":212,"Ĕ":213,"ĕ":214,"Ė":215,"ė":216,"Ę":217,"ę":218,"Ě":219,"ě":220,"Ĝ":221,"ĝ":222,"Ğ":223,"ğ":224,"Ġ":225,"ġ":226,"Ģ":227,"ģ":228,"Ĥ":229,"ĥ":230,"Ħ":231,"ħ":232,"Ĩ":233,"ĩ":234,"Ī":235,"ī":236,"Ĭ":237,"ĭ":238,"Į":239,"į":240,"İ":241,"ı":242,"IJ":243,"ij":244,"Ĵ":245,"ĵ":246,"Ķ":247,"ķ":248,"ĸ":249,"Ĺ":250,"ĺ":251,"Ļ":252,"ļ":253,"Ľ":254,"ľ":255,"Ŀ":256,"ŀ":257,"Ł":258,"ł":259,"Ń":260,"ra":261,"an":262,"á":263,"ba":264,"ca":265,"ua":266,"é":267,"ç":268,"ru":269,"ana":270,"Ca":271,"im":272,"Ara":273,"na":274,"ri":275,"ti":276,"ap":277,"ê":278,"ÃŃ":279,"Am":280,"ara":281,"ã":282,"Aba":283,"Ġ(":284,"Ja":285,"uara":286,"au":287,"ré":288,"uru":289,"mb":290,"ng":291,"rá":292,"ça":293,"An":294,"Ti":295,"ira":296,"ma":297,"ta":298,"Ka":299,"bi":300,"bu":301,"cu":302,"ni":303,"ui":304,"va":305,"wa":306,"Ana":307,"bá":308,"et":309,"he":310,"api":311,"nga":312,"Ai":313,"Aj":314,"Ju":315,"Ma":316,"Man":317,"Para":318,"eba":319,"gua":320,"mim":321,"oca":322,"quara":323,"uã":324,"ú":325,"Cari":326,"Amana":327,"Ap":328,"Aca":329,"Aim":330,"Ira":331,"Ita":332,"Juru":333,"Ya":334,"rê":335,"ss":336,"ue":337,"uç":338,"xana":339,"and":340,"ang":341,"anã":342,"Arac":343,"auá":344,"çaÃŃ":345,"apixana":346,"Ab":347,"Ay":348,"Awa":349,"Gua":350,"Guara":351,"Nhe":352,"Oca":353,"Pe":354,"Pi":355,"Tu":356,"Xim":357,"ará":358,"cá":359,"di":360,"eru":361,"emb":362,"it":363,"ju":364,"ka":365,"mu":366,"mÃŃ":367,"oti":368,"pi":369,"po":370,"ry":371,"tê":372,"uça":373,"ó":374,"camim":375,"çu":376,"ÃŃba":377,"Abaca":378,"biquara":379,"buré":380,"Maca":381,"Mani":382,"guar":383,"úna":384,"Tupi":385,"Av":386,"Ati":387,"Ava":388,"Da":389,"Ia":390,"Iba":391,"Ibi":392,"Jap":393,"Lau":394,"Mu":395,"Mana":396,"Puç":397,"Pará":398,"Poti":399,"Qui":400,"Ru":401,"Tim":402,"Tap":403,"Ua":404,"Wa":405,"Xu":406,"aÃŃ":407,"bo":408,"boca":409,"ce":410,"cl":411,"cé":412,"cê":413,"eq":414,"enga":415,"erê":416,"guara":417,"ia":418,"ita":419,"jé":420,"man":421,"mana":422,"nd":423,"pauá":424,"saÃŃ":425,"tã":426,"uca":427,"uri":428,"uet":429,"yu":430,"õ":431,"ü":432,"anhe":433,"anbiquara":434,"bara":435,"baquara":436,"uar":437,"ém":438,"çara":439,"runa":440,"Cai":441,"Camu":442,"Arap":443,"Arara":444,"tir":445,"tiba":446,"Ama":447,"Amand":448,"Abaet":449,"Jacu":450,"Java":451,"auim":452,"urui":453,"mbeba":454,"Anh":455,"Tiju":456,"Tiyu":457,"buna":458,"uia":459,"Aju":460,"ParaÃŃba":461,"angü":462,"Guará":463,"Guarani":464,"embé":465}