feat: hf dataset connection

.github/workflows/sync_dataset_hf.yml

@@ -0,0 +1,27 @@
+name: Sync Hugging Face Dataset
+
+on:
+  schedule:
+    - cron: '0 * * * *'  # Runs every hour
+
+jobs:
+  sync_dataset:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Sync Datasets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: python sync_dataset_hf.py

app/display.py

@@ -12,7 +12,6 @@ HEADERS = ["Identifier", "Location", "Wounded", "Dead"]
 
 
 def save_display_individual(gallery, df, error_box, data):
-    #print(data)
     individual, error_box, data = validate_save_individual(data, error_box)
     if individual:
         all_animals = get_json_all_individuals()

app/main.py

@@ -20,6 +20,12 @@ from styling.theme import css
 
 from geolocalisation.js_geolocation import js_geocode, display_location
 
+from datasets import disable_caching
+disable_caching()
+
+dataset_id = "SDSC/digiwild-dataset"
+data_files = "data/train-00000-of-00001.parquet"
+
 # with gr.Blocks(theme=theme, css=css) as demo:
 with gr.Blocks(theme='shivi/calm_seafoam') as demo:
     individual = gr.State({})
@@ -396,6 +402,7 @@ with gr.Blocks(theme='shivi/calm_seafoam') as demo:
     show_modal.click(lambda: Modal(visible=True), None, modal)
     show_modal.click(create_json_one_individual)
     show_modal.click(create_tmp)
+
     #submit_button.click(save_and_rest_df, inputs=[df], outputs=[df])
 
 

app/sync_dataset_hf.py

@@ -0,0 +1,35 @@
+from datasets import load_dataset, DownloadMode
+import json
+import os 
+from huggingface_hub import HfApi , hf_hub_download
+
+dataset_id = "SDSC/digiwild-dataset"
+token = os.getenv("HUGGINGFACE_TOKEN")
+
+# Initialize API client
+api = HfApi(token=token)
+
+# Load all metadata files
+files = api.list_repo_files(dataset_id, repo_type="dataset")
+json_files = [file for file in files if file.endswith(".json")]
+
+# Load the metadata compilation 
+try: 
+    data_files = "data/train-00000-of-00001.parquet"
+    metadata = load_dataset(
+                            dataset_id, 
+                            data_files=data_files)
+    # Add new json entries to dataset 
+    for file in json_files: 
+        file = hf_hub_download(repo_id=dataset_id, filename=file, repo_type="dataset")
+        with open(file, "r") as f:
+            new = json.load(f)
+        if not(new["image_md5"] in metadata["train"]["image_md5"]):
+            metadata["train"] = metadata["train"].add_item(new)
+except: 
+    metadata = load_dataset(
+                    dataset_id,
+                    data_files=json_files)
+
+
+metadata.push_to_hub(dataset_id, token=token)

app/validation_submission/create_json.py

@@ -1,6 +1,19 @@
 import json 
 
+import random
+import string
+
+import hashlib
+
+def generate_random_md5():
+    # Generate a random string
+    random_string = ''.join(random.choices(string.ascii_letters + string.digits, k=16))
+    # Encode the string and compute its MD5 hash
+    md5_hash = hashlib.md5(random_string.encode()).hexdigest()
+    return md5_hash
+
 def create_json_one_individual(one_individual={}):
+    one_individual["image_md5"] = generate_random_md5()
     one_individual = json.dumps(one_individual)
     with open("data/one_individual.json", "w") as outfile:
         outfile.write(one_individual)

app/validation_submission/get_json.py

@@ -5,9 +5,13 @@ def get_json_one_individual():
         one_individual = json.load(openfile)
     return one_individual
 
+## TO DO : check this works
+import os
 def get_json_all_individuals():
-    with open("data/all_individuals.json", "r") as openfile:
-        all_individuals = json.load(openfile)
+    all_animals = os.getfiles("data")
+    all_individuals = []
+    for animal in all_animals: 
+        all_individuals.append(animal)
     return all_individuals
 
 def get_json_tmp(tmp_name):

app/validation_submission/submission.py

@@ -14,4 +14,20 @@ def save_to_all_individuals(one_individual):
     all_individuals_for_json = json.dumps(all_individuals)
     with open("data/all_individuals.json", "w") as outfile:
         outfile.write(all_individuals_for_json)
-    return all_individuals
+    return all_individuals
+
+from huggingface_hub import HfApi
+import os 
+
+#save all individuals one by one in JSON wish md5 hash as json name 
+def push_to_dataset_hf():
+    token = os.environ.get("HF_TOKEN", None)
+    api = HfApi(token=token)
+    with open("data/all_individuals.json", "r") as f:
+            all = json.load(f)
+    api.upload_file(
+        path_or_fileobj=f.name,
+        path_in_repo=path_in_repo,
+        repo_id="SDSC/digiwild-dataset",
+        repo_type="dataset",
+    )

requirements.txt

@@ -3,4 +3,7 @@ gradio_modal
 geopy
 geopandas
 pillow
-python-dotenv
+python-dotenv
+datasets
+huggingface_hub
+hashlib