Duplicate from dvilasuero/alpaca-cleaned-de

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,7 @@
+FROM argilla/argilla-quickstart:latest
+
+COPY load_data.py /
+
+RUN pip install argilla[listeners]
+
+CMD whoami && /start_quickstart_argilla.sh

README.md

@@ -0,0 +1,13 @@
+---
+title: Alpaca Dataset Validation with Argilla
+emoji: 🦙 🏷️
+colorFrom: purple
+colorTo: red
+sdk: docker
+app_port: 6900
+fullWidth: true
+tags:
+- argilla
+- somosnlp
+duplicated_from: dvilasuero/alpaca-cleaned-de
+---

load_data.py

@@ -0,0 +1,105 @@
+import sys
+import time
+import os
+
+import argilla as rg
+import pandas as pd
+import requests
+from datasets import load_dataset, concatenate_datasets
+
+from argilla.listeners import listener
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+SOURCE_DATASET = "LEL-A/translated_german_alpaca"
+RG_DATASET_NAME = "translated-german-alpaca"
+HUB_DATASET_NAME = os.environ.get('HUB_DATASET_NAME', f"{SOURCE_DATASET}_validation")
+
+@listener(
+    dataset=RG_DATASET_NAME, 
+    query="status:Validated", # https://docs.argilla.io/en/latest/guides/features/queries.html
+    execution_interval_in_seconds=1200, # interval to check the execution of `save_validated_to_hub`
+)
+def save_validated_to_hub(records, ctx):
+    if len(records) > 0:
+        ds = rg.DatasetForTextClassification(records=records).to_datasets()   
+        if HF_TOKEN:
+            print("Pushing the dataset")
+            print(ds)
+            ds.push_to_hub(HUB_DATASET_NAME, token=HF_TOKEN)
+        else:
+            print("SET HF_TOKEN and HUB_DATASET_NAME TO SYNC YOUR DATASET!!!")
+    else:
+        print("NO RECORDS found")
+
+class LoadDatasets:
+    def __init__(self, api_key, workspace="team"):
+        rg.init(api_key=api_key, workspace=workspace)
+
+    @staticmethod
+    def load_somos():
+        # Leer el dataset del Hub
+        try:
+            print(f"Trying to sync with {HUB_DATASET_NAME}")
+            old_ds = load_dataset(HUB_DATASET_NAME, split="train")
+        except Exception as e:
+            print(f"Not possible to sync with {HUB_DATASET_NAME}")
+            print(e)
+            old_ds = None
+
+        print(f"Loading dataset: {SOURCE_DATASET}")
+        dataset = load_dataset(SOURCE_DATASET, split="train")
+    
+        
+        if old_ds:
+            print("Concatenating datasets")
+            dataset = concatenate_datasets([dataset, old_ds])
+            print("Concatenated dataset is:")
+            print(dataset)
+            
+        dataset = dataset.remove_columns("metrics")
+        records = rg.DatasetForTextClassification.from_datasets(dataset)
+
+        settings = rg.TextClassificationSettings(
+            label_schema=["BAD INSTRUCTION", "BAD INPUT", "BAD OUTPUT", "INAPPROPRIATE", "BIASED", "ALL GOOD"]
+        )
+        
+        print(f"Configuring dataset: {RG_DATASET_NAME}")
+        rg.configure_dataset(name=RG_DATASET_NAME, settings=settings, workspace="team")
+        
+        # Log the dataset
+        print(f"Logging dataset: {RG_DATASET_NAME}")
+        rg.log(
+            records,
+            name=RG_DATASET_NAME,
+            tags={"description": "Alpaca dataset to clean up"},
+            batch_size=200
+        )
+        
+        # run listener
+        save_validated_to_hub.start()
+
+if __name__ == "__main__":
+    API_KEY = sys.argv[1]
+    LOAD_DATASETS = sys.argv[2]
+
+    if LOAD_DATASETS.lower() == "none":
+        print("No datasets being loaded")
+    else:
+        while True:
+            try:
+                response = requests.get("http://0.0.0.0:6900/")
+                if response.status_code == 200:
+                    ld = LoadDatasets(API_KEY)
+                    ld.load_somos()
+                    break
+
+            except requests.exceptions.ConnectionError:
+                pass
+            except Exception as e:
+                print(e)
+                time.sleep(10)
+                pass
+
+            time.sleep(5)
+    while True:
+        time.sleep(60)