feat: fix load from huggingface

app.py

@@ -146,6 +146,7 @@ dynamic_dataset = None
 def load_dataset():
     global dynamic_dataset
     try:
+        print('load_dataset')
         with open("data/annotated_data.json", 'rt') as dataset:
             ANNOTATED_DATA = json.load(dataset)
         dynamic_dataset = DynamicDataset(ANNOTATED_DATA)
@@ -530,11 +531,25 @@ def convert_hf_dataset_to_ner_format(dataset):
     
     return converted_data
 
-def load_from_huggingface(dataset_name: str, split: str = "all"):
+def load_from_huggingface(dataset_name: str):
     """Load dataset from Hugging Face Hub"""
     try:
-        dataset = load_dataset(dataset_name, split=split)
-        converted_data = convert_hf_dataset_to_ner_format(dataset)
+        # Download the JSON file from Hugging Face
+        import requests
+        import json
+        
+        # Construct the raw URL for the JSON file
+        raw_url = f"https://huggingface.co/datasets/{dataset_name}/raw/main/annotated_data.json"
+        
+        # Download the file
+        response = requests.get(raw_url)
+        if response.status_code == 200:
+            print('response status', response.status_code)
+            print('response', response.text)
+            dataset = json.loads(response.text)
+            converted_data = dataset  # Data is already in the correct format
+        else:
+            raise Exception(f"Failed to download dataset: {response.status_code}")
         
         # Save the converted data
         os.makedirs("data", exist_ok=True)
@@ -543,7 +558,8 @@ def load_from_huggingface(dataset_name: str, split: str = "all"):
             
         return f"Successfully loaded and converted dataset: {dataset_name}"
     except Exception as e:
-        return f"Error loading dataset: {str(e)}"
+        error_msg = f"Error loading dataset: {str(e)}"
+        return error_msg
 
 def load_from_local_file(file_path: str, file_format: str = "json"):
     """Load and convert data from local file in various formats"""
@@ -891,14 +907,7 @@ with gr.Blocks() as demo:
                                 placeholder="Enter dataset name (e.g., conll2003)",
                                 scale=3
                             )
-                            dataset_split = gr.Dropdown(
-                                choices=["train", "validation", "test"],
-                                value="train",
-                                label="Dataset Split",
-                                scale=2
-                            )
                             load_dataset_btn = gr.Button("Load Dataset", scale=1)
-                        hf_status = gr.Textbox(label="Dataset Loading Status")
                     
                     bar = gr.Slider(
                         minimum=0,
@@ -1002,16 +1011,17 @@ with gr.Blocks() as demo:
                         outputs=[inp_box, bar]
                     )
                     
-                    def load_hf_dataset(name, split):
-                        status = load_from_huggingface(name, split)
+                    def load_hf_dataset(name):
+                        status = load_from_huggingface(name)
+                        print('status', status)
                         if "Successfully" in status:
-                            return load_dataset(), status
-                        return [status], 0, 0, status
+                            return load_dataset()
+                        return [("Error loading dataset: " + status, None)], gr.update(value=0, maximum=1)
                     
                     load_dataset_btn.click(
                         fn=load_hf_dataset,
-                        inputs=[dataset_name, dataset_split],
-                        outputs=[inp_box, bar, hf_status]
+                        inputs=[dataset_name],
+                        outputs=[inp_box, bar]
                     )
                     
                     apply_btn.click(fn=update_example, inputs=inp_box, outputs=inp_box)