test commitscheduler

app.py

@@ -6,6 +6,7 @@ import json
 import os
 from huggingface_hub import HfApi, CommitScheduler
 import uuid
+from src.components import ParquetScheduler
 
 
 HF_API_KEY = os.environ.get("HF_TOKEN", None)
@@ -19,14 +20,21 @@ submissions_folder = "submissions"
 submissions_file = os.path.join(submissions_folder, f"submissions_{uuid.uuid4()}.json")
 os.makedirs(submissions_folder, exist_ok=True)
 
-scheduler = CommitScheduler(
-    token=HF_API_KEY,
-    repo_id=REPO_ID,
-    repo_type="dataset",
-    folder_path=submissions_folder,
-    path_in_repo="submissions",
-    every=1,
-)
+# scheduler = CommitScheduler(
+#     token=HF_API_KEY,
+#     repo_id=REPO_ID,
+#     repo_type="dataset",
+#     folder_path=submissions_folder,
+#     path_in_repo="submissions",
+#     every=1,
+# )
+
+# Define the ParquetScheduler instance with your repo details
+scheduler = ParquetScheduler(repo_id=REPO_ID, 
+                             token=HF_API_KEY, every=1, 
+                             path_in_repo=submissions_folder, 
+                             repo_type="dataset")
+
 
 def load_data(repo_id):
     dataset = load_dataset(f'{repo_id}', name='sentences', split='sentences')
@@ -64,13 +72,17 @@ def store_submission(api: HfApi, sentence: str, translation: str, translation_fr
     #             repo_type="dataset",
     #         )
 
-    with scheduler.lock:
-        with submissions_file.open("a") as f:
-            f.write(json.dumps({
-                "darija": translation_fr, 
+    # with scheduler.lock:
+    #     with submissions_file.open("a") as f:
+    #         f.write(json.dumps({
+    #             "darija": translation_fr, 
+    #             "eng": translation, 
+    #             "darija_ar": sentence}))
+    #         f.write("\n")
+
+    scheduler.append({"darija": translation_fr, 
                 "eng": translation, 
-                "darija_ar": sentence}))
-            f.write("\n")
+                "darija_ar": sentence})
 
     st.success(
         f"""Translation submitted successfully to