Fixing datatypes so it can be visualized

main.py

@@ -65,6 +65,7 @@ def main(date_to_fetch):
 
         # Figure out dates when we restart
         old_data_most_recent_date = old_data['date'].max()
+        old_data_most_recent_date = datetime.strptime(old_data_most_recent_date, '%Y-%m-%d').date()
         most_recent_date = max(old_data_most_recent_date, most_recent_date)
 
         if len(old_data) == len(new_data):
@@ -73,6 +74,9 @@ def main(date_to_fetch):
 
         # Convert back to dataset
         dataset["all_days"] = Dataset.from_pandas(new_data)
+
+        # Update README
+        update_readme(dataset_name, subreddit, date_to_fetch)
     else:
         logger.debug("Creating new split 'all_days'")
         dataset["all_days"] = Dataset.from_pandas(df)
@@ -81,7 +85,6 @@ def main(date_to_fetch):
 
     # Push the augmented dataset to the Hugging Face hub
     logger.debug(f"Pushing data for {date_to_fetch} to the Hugging Face hub")
-    update_readme(dataset_name, subreddit, date_to_fetch)
     dataset.push_to_hub(dataset_name, token=auth_token)
     logger.info(f"Processed and pushed data for {date_to_fetch} to the Hugging Face Hub")
     return most_recent_date

utilities/pushshift_data.py

@@ -149,8 +149,8 @@ def submissions_to_dataframe(submissions: List[Dict[str, Any]]) -> pd.DataFrame:
     df = df[cols]
     # Convert the "created_utc" column to a datetime column with timezone information
     df['created_utc'] = pd.to_datetime(df['created_utc'], unit='s').dt.tz_localize('UTC')
-    df['date'] = df['created_utc'].dt.date
-    df['time'] = df['created_utc'].dt.time
+    df['date'] = df['created_utc'].dt.date.astype(str)
+    df['time'] = df['created_utc'].dt.time.astype(str)
     return df