Upload folder using huggingface_hub (#3)

- Upload folder using huggingface_hub (76cdab0b84c05d4d090700242c31104b36c61425)


Co-authored-by: FBL <fblgit@users.noreply.huggingface.co>

.gitattributes

@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+saves/huggyllama-llama-7b/Rowan-hellaswag/neighbors_dls.txt filter=lfs diff=lfs merge=lfs -text
+saves/huggyllama-llama-7b/cais-mmlu/neighbors_dls.txt filter=lfs diff=lfs merge=lfs -text
+saves/huggyllama-llama-7b/gsm8k/neighbors_dls.txt filter=lfs diff=lfs merge=lfs -text
+saves/mistralai-Mistral-7B-v0.1/Rowan-hellaswag/neighbors_dls.txt filter=lfs diff=lfs merge=lfs -text
+saves/mistralai-Mistral-7B-v0.1/cais-mmlu/neighbors_dls.txt filter=lfs diff=lfs merge=lfs -text
+saves/mistralai-Mistral-7B-v0.1/gsm8k/neighbors_dls.txt filter=lfs diff=lfs merge=lfs -text

data/code_eval_board.csv

@@ -5,7 +5,6 @@ T,Models,ARC,HellaSwag,MMLU,TruthfulQA,Winogrande,GSM8K,Reference Model
 🟢,mistralai/Mistral-7B-v0.1,0.54,0.51,0.46,0.75,0,0.91,huggyllama/llama-7b
 🔶,rishiraj/meow,0.11,0.49,0.28,0.36,0.02,0.95,huggyllama/llama-7b
 🔶,Q-bert/MetaMath-Cybertron-Starling,0.52,0.64,0.51,0.75,0.01,0.99,huggyllama/llama-7b
-🔶,upstage/SOLAR-10.7B-Instruct-v1.0,0.11,0.49,0.28,0.36,0.01,0.96,huggyllama/llama-7b
 🔶,AIDC-ai-business/Marcoroni-7B-v3,0.1,0.14,0.2,0.41,0.0,0.95,mistralai/Mistral-7B-v0.1
 🔶,amazon/MistralLite,0.09,0.14,0.2,0.43,0.0,0.73,mistralai/Mistral-7B-v0.1
 🔶,openchat/openchat_3.5,0.13,0.13,0.23,0.45,0.0,0.97,mistralai/Mistral-7B-v0.1
@@ -14,7 +13,6 @@ T,Models,ARC,HellaSwag,MMLU,TruthfulQA,Winogrande,GSM8K,Reference Model
 🔶,microsoft/Orca-2-7b,0.88,0.8,0.77,0.91,0.0,1.0,mistralai/Mistral-7B-v0.1
 🔶,WizardLM/WizardMath-7B-V1.1,0.1,0.11,0.21,0.4,0.0,0.99,mistralai/Mistral-7B-v0.1
 🔶,01-ai/Yi-6B-200K,0.19,0.3,0.3,0.6,0.0,0.93,mistralai/Mistral-7B-v0.1
-🔶,Intel/neural-chat-7b-v3-3,0.06,0.15,0.18,0.47,0.0,0.98,mistralai/Mistral-7B-v0.1
 🔶,mistralai/Mistral-7B-Instruct-v0.2,0.06,0.21,0.17,0.48,0.0,0.95,mistralai/Mistral-7B-v0.1
 🔶,Yhyu13/LMCocktail-10.7B-v1,0.1,0.44,0.23,0.51,0.0,0.97,mistralai/Mistral-7B-v0.1
 🔶,ehartford/dolphin-2.1-mistral-7b,0.08,0.1,0.2,0.4,0.0,0.92,mistralai/Mistral-7B-v0.1
@@ -34,11 +32,14 @@ T,Models,ARC,HellaSwag,MMLU,TruthfulQA,Winogrande,GSM8K,Reference Model
 🟢,huggyllama/llama-7b,0.11,0.17,0.22,0.46,0.0,0.79,mistralai/Mistral-7B-v0.1
 🟢,tiiuae/falcon-7b-instruct,0.06,0.16,0.19,0.56,0.0,0.98,mistralai/Mistral-7B-v0.1
 🔶,NousResearch/Nous-Hermes-llama-2-7b,0.09,0.18,0.26,0.5,0.0,0.96,mistralai/Mistral-7B-v0.1
+🔶,openaccess-ai-collective/DPOpenHermes-7B-v2,0.08,0.11,0.22,0.41,0.0,0.96,mistralai/Mistral-7B-v0.1
+🟢,01-ai/Yi-6B,0.28,0.32,0.3,0.62,0.02,0.94,mistralai/Mistral-7B-v0.1
 🔶,Intel/neural-chat-7b-v3-1,0.1,0.15,0.18,0.49,0.0,0.81,mistralai/Mistral-7B-v0.1
 🔶,fblgit/juanako-7b-UNA,0.09,0.15,0.18,0.46,0.0,0.81,mistralai/Mistral-7B-v0.1
 🔶,Intel/neural-chat-7b-v3-2,0.12,0.14,0.2,0.5,0.0,0.93,mistralai/Mistral-7B-v0.1
 🔶,fblgit/una-cybertron-7b-v2-bf16,0.1,0.12,0.21,0.46,0.0,0.92,mistralai/Mistral-7B-v0.1
+🔶,Intel/neural-chat-7b-v3-3,0.06,0.15,0.18,0.47,0.0,0.98,mistralai/Mistral-7B-v0.1
 🔶,fblgit/una-cybertron-7b-v3-OMA,0.04,0.16,0.17,0.36,0.0,0.94,mistralai/Mistral-7B-v0.1
+🔶,fblgit/una-xaberius-34b-v1beta,0.37,0.54,0.33,0.61,0.04,0.96,huggyllama/llama-7b
+🔶,upstage/SOLAR-10.7B-Instruct-v1.0,0.11,0.49,0.28,0.36,0.01,0.96,huggyllama/llama-7b
 🔶,VAGOsolutions/SauerkrautLM-SOLAR-Instruct,0.12,0.54,0.32,0.34,0.01,0.96,huggyllama/llama-7b
-🔶,openaccess-ai-collective/DPOpenHermes-7B-v2,0.08,0.11,0.22,0.41,0.0,0.96,mistralai/Mistral-7B-v0.1
-🟢,01-ai/Yi-6B,0.28,0.32,0.3,0.62,0.02,0.94,mistralai/Mistral-7B-v0.1

data/queue.csv

@@ -1,6 +1,10 @@
 Type,Model,ref_model
+🔶 finetuned,openaccess-ai-collective/DPOpenHermes-7B-v2,mistralai/Mistral-7B-v0.1
+🔶 finetuned,microsoft/Orca-2-7B,mistralai/Mistral-7B-v0.1
+🟢 base,01-ai/Yi-6B,mistralai/Mistral-7B-v0.1
 🔶 finetuned,VAGOsolutions/SauerkrautLM-SOLAR-Instruct,mistralai/Mistral-7B-v0.1
+🔶 finetuned,VAGOsolutions/SauerkrautLM-SOLAR-Instruct,huggyllama/llama-7b
 🔶 finetuned,VAGOsolutions/SauerkrautLM-SOLAR-Instruct,upstage/SOLAR-10.7B-v1.0
 🔶 finetuned,Sao10K/SOLAR-10.7B-NahIdWin,upstage/SOLAR-10.7B-v1.0
 🔶 finetuned,fblgit/una-xaberius-34b-v1beta,huggyllama/llama-7b
-🔶 finetuned,Locutusque/Orca-2-13b-SFT-v4,upstage/SOLAR-10.7B-v1.0
+🔶 finetuned,fblgit/una-cybertron-7b-v2-bf16,mistralai/Mistral-7B-v0.1

detect-pretrain-code-contamination/src/eval.py

@@ -154,6 +154,7 @@ def process_gsm8k(data):
     return new_data
 
 def process_winogrande(data):
+    '''
     new_data = []
     for ex in data:
         new_ex = {}
@@ -163,6 +164,31 @@ def process_winogrande(data):
         new_ex["input"] = ex["sentence"] + " " + output
         new_data.append(new_ex)
     return new_data
+    '''
+    new_data = []
+    for doc in data:
+        new_doc = {}
+
+        # Convert the answer to a numeric index
+        answer_to_num = {"1": 0, "2": 1}
+        label_idx = answer_to_num[doc["answer"]]
+
+        # Generate options and select the correct one based on label_idx
+        options = [doc["option1"], doc["option2"]]
+        output = options[label_idx]
+
+        # Build the new sentence by inserting the selected option
+        idx = doc["sentence"].index("_")
+        input_sentence = doc["sentence"][:idx] + output + doc["sentence"][idx+1:]
+
+        # Assigning the processed values to the new_doc
+        new_doc["output"] = output
+        new_doc["input"] = input_sentence
+
+        # Append the processed document to new_data
+        new_data.append(new_doc)
+
+    return new_data
 # I'm not sure if that's the correct format for winogrande given how the dataset works.
 
 def process_hellaswag(data):

detect-pretrain-code-contamination/src/run.py

@@ -228,7 +228,7 @@ def main(target_model,ref_model,output_dir,data,length,key_name,ratio_gen):
         data = convert_huggingface_data_to_list_dic(dataset)
         data = process_hellaswag(data)
     elif data == "winogrande": 
-        dataset = load_dataset(data,"winogrande_debiased", split="validation")
+        dataset = load_dataset(data, "winogrande_xl", split="validation")
         data = convert_huggingface_data_to_list_dic(dataset)
         data = process_winogrande(data)
 

saves/huggyllama-llama-7b/Rowan-hellaswag/neighbors_dls.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:515612955fcea9bbbd4af7c8638587f24d3719ba9fd7e7f9f25fadf6df4f4247
-size 1499897
+oid sha256:e208e3e34929e172cbf326bce491dea65acaeb68cda298ad88c9f3458dd1141c
+size 1500377

saves/huggyllama-llama-7b/cais-mmlu/neighbors_dls.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e587e8ab0615640d50853b34db158e09955ed9de86f1ac1cb0fd1e50ac2e76b
-size 1393126
+oid sha256:bc89b10856608d5fad800c42dbf6256aa6e9da35b00d3717e6e9ec8c14abae36
+size 1388794

saves/huggyllama-llama-7b/gsm8k/neighbors_dls.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1af9f4a69ef5940b4751928ecc12c78873933a4558fee2f3da956f40f778ec88
-size 1945223
+oid sha256:2e9ad004f0004acf56d36f64003af2fab95fae92db9ee3d20bc630bc8aec993f
+size 1913508

saves/mistralai-Mistral-7B-v0.1/Rowan-hellaswag/neighbors_dls.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d3b8b02ba25208dcc460a1dc6280b51ca12450a4ad9ce2dfe2e3c8321d83ec57
-size 1552744
+oid sha256:1aaa6227f351c4c721147c4cf3c5aedd84ad29ab994ac610585c1a4dea472770
+size 1550973

saves/mistralai-Mistral-7B-v0.1/cais-mmlu/neighbors_dls.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ba712896a82071db46209d30bbf37569f79ea3a13185d0b8507192dd83cb491
-size 1397452
+oid sha256:e302914aa8b79a47b2e5ca6335e9d8e64dc9bc7a366e2907efb8e7f7b611eff9
+size 1396480

saves/mistralai-Mistral-7B-v0.1/gsm8k/neighbors_dls.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83a28debd00668ff79a5cc6978ee8661c7d1d0492691e766d4dca03292f5410f
-size 1838050
+oid sha256:a60198c9a72aaf9927c7e2b6d79a07c7a8a41c9f8d27a12c746f9b738fecfa9c
+size 1841218