:tada: init

app.py

@@ -2,7 +2,7 @@ import gradio as gr
 import torch
 from diffusers import DiffusionPipeline
 from transformers import AutoTokenizer, CLIPTokenizerFast, T5TokenizerFast
-
+import pandas as pd
 
 def load_tokenizers(model_id: str) -> list[CLIPTokenizerFast | T5TokenizerFast | None]:
     config = DiffusionPipeline.load_config(model_id)
@@ -25,19 +25,20 @@ def load_tokenizers(model_id: str) -> list[CLIPTokenizerFast | T5TokenizerFast |
 
 
 @torch.no_grad()
-def inference(model_id: str, input_text: str):
+def inference(model_id: str, text: str):
     tokenizers = load_tokenizers(model_id)
 
     text_pairs_components = []
     special_tokens_components = []
+    tokenizer_details_components = []
     for i, tokenizer in enumerate(tokenizers):
         if tokenizer:
             label_text = f"Tokenizer {i + 1}: {tokenizer.__class__.__name__}"
 
             # テキストとトークンIDのペアを作成
             input_ids = tokenizer(
-                text=input_text,
-                truncation=True,
+                text=text,
+                truncation=False,
                 return_length=False,
                 return_overflowing_tokens=False,
             ).input_ids
@@ -49,7 +50,6 @@ def inference(model_id: str, input_text: str):
                 label=label_text,
                 value=token_pairs,
                 visible=True,
-                show_legend=True,
             )
 
             # スペシャルトークンを追加
@@ -63,16 +63,32 @@ def inference(model_id: str, input_text: str):
                 label=label_text,
                 value=special_tokens,
                 visible=True,
-                show_legend=True,
+            )
+
+            # トークナイザーの詳細情報を追加
+            tokenizer_details = pd.DataFrame([
+                ("Type", tokenizer.__class__.__name__),
+                ("Vocab Size", tokenizer.vocab_size),
+                ("Model Max Length", tokenizer.model_max_length),
+                ("Padding Side", tokenizer.padding_side),
+                ("Truncation Side", tokenizer.truncation_side),
+            ], columns=["Attribute", "Value"])
+            output_tokenizer_details = gr.Dataframe(
+                headers=["Attribute", "Value"],
+                value=tokenizer_details,
+                label=label_text,
+                visible=True,
             )
         else:
             output_text_pair_component = gr.HighlightedText(visible=False)
             output_special_tokens_component = gr.HighlightedText(visible=False)
+            output_tokenizer_details = gr.Dataframe(visible=False)
 
         text_pairs_components.append(output_text_pair_component)
         special_tokens_components.append(output_special_tokens_component)
+        tokenizer_details_components.append(output_tokenizer_details)
 
-    return text_pairs_components + special_tokens_components
+    return  text_pairs_components + special_tokens_components + tokenizer_details_components
 
 
 if __name__ == "__main__":
@@ -110,6 +126,11 @@ if __name__ == "__main__":
                     output_special_tokens_1 = gr.HighlightedText()
                     output_special_tokens_2 = gr.HighlightedText()
                     output_special_tokens_3 = gr.HighlightedText()
+            with gr.Tab(label="Tokenizer Details"):
+                with gr.Column():
+                    output_tokenizer_details_1 = gr.Dataframe(headers=["Attribute", "Value"])
+                    output_tokenizer_details_2 = gr.Dataframe(headers=["Attribute", "Value"])
+                    output_tokenizer_details_3 = gr.Dataframe(headers=["Attribute", "Value"])
 
             with gr.Row():
                 clear_button = gr.ClearButton(components=[input_text])
@@ -123,6 +144,9 @@ if __name__ == "__main__":
             output_special_tokens_1,
             output_special_tokens_2,
             output_special_tokens_3,
+            output_tokenizer_details_1,
+            output_tokenizer_details_2,
+            output_tokenizer_details_3,
         ]
         submit_button.click(fn=inference, inputs=all_inputs, outputs=all_output)
 
@@ -141,4 +165,4 @@ if __name__ == "__main__":
             cache_examples=True,
         )
 
-    demo.queue().launch()
+    demo.queue().launch()