fix HuggingFaceDatasetSaver

app.py

@@ -4,6 +4,7 @@ import numpy as np
 import fasttext
 import os
 import urllib
+import huggingface_hub
 from transformers import MBartForConditionalGeneration, MBart50Tokenizer
 
 
@@ -12,10 +13,46 @@ MODEL_URL_MUL_MYV = 'slone/mbart-large-51-mul-myv-v1'
 MODEL_URL_LANGID = 'https://huggingface.co/slone/fastText-LID-323/resolve/main/lid.323.ftz'
 MODEL_PATH_LANGID = 'lid.323.ftz'
 
+
+class DSaverFixed(gr.HuggingFaceDatasetSaver):
+    def __init__(
+        self,
+        hf_token: str,
+        dataset_name: str,
+        organization=None,
+        private=False,
+    ):
+        self.hf_token = hf_token
+        self.dataset_name = dataset_name
+        self.organization_name = organization
+        self.dataset_private = private
+
+    def setup(self, components, flagging_dir):
+        path_to_dataset_repo = huggingface_hub.create_repo(
+            repo_id=self.dataset_name,
+            token=self.hf_token,
+            private=self.dataset_private,
+            repo_type="dataset",
+            exist_ok=True,
+        )
+        self.path_to_dataset_repo = path_to_dataset_repo
+        self.components = components
+        self.flagging_dir = flagging_dir
+        self.dataset_dir = os.path.join(flagging_dir, self.dataset_name)
+        self.repo = huggingface_hub.Repository(
+            local_dir=self.dataset_dir,
+            clone_from=path_to_dataset_repo,
+            use_auth_token=self.hf_token,
+        )
+        self.repo.git_pull(lfs=True)
+        self.infos_file = os.path.join(self.dataset_dir, "dataset_infos.json")
+
+
 HF_TOKEN = os.getenv('HF_TOKEN')
-hf_writer = gr.HuggingFaceDatasetSaver(
-    HF_TOKEN,
-    dataset_name="slone/myv-translation-2022-demo-flags",
+hf_writer = DSaverFixed(
+    hf_token=HF_TOKEN,
+    dataset_name="myv-translation-2022-demo-flags",
+    organization="slone",
     private=True,
 )