Upload 8 files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,40 @@
+import csv 
+import xml.etree.ElementTree as ET 
+import glob, os
+
+rootFolder = "c:/317"
+
+file = open(rootFolder + "/result.csv", "w", encoding="utf-8")
+file.write("prompt,text,rejected_text\n")
+
+def parseXML(xmlFile):
+
+    prompt = xmlFile.replace("Using_", "").replace(".xml", "").replace(".", " ").replace("_", " ")
+    text  = ""
+
+    try:
+        tree = ET.parse(rootFolder + "/" + xmlFile)
+        root = tree.getroot()
+
+        for item in root.findall(".//text"):
+            text += item.text
+
+        if text.find("а") == -1:
+            #file.write("### prompt\n")
+            file.write(prompt + "," + text.replace(",", " ") + "\n")
+            #file.write("### text\n")
+            #file.write(text.replace(",", " "))
+
+    except:
+        print("=======")
+
+
+os.chdir(rootFolder)
+for xmlFile in glob.glob("*.xml"):
+    print(xmlFile)
+    parseXML(xmlFile)
+
+
+## parseXML('Using_WinRT_Viewer.Search_Panel.xml')
+
+file.close()

app2.py

@@ -0,0 +1,16 @@
+import torch
+
+device = torch.device("cuda")
+
+tenz = torch.tensor([1.,2.], device=device)
+#tenz.toDevice(device)
+
+print(torch.cuda.is_available())
+
+from datasets import Dataset
+
+dataset = Dataset.from_dict({"a": [0, 1, 2]})
+dataset_with_duplicates = dataset.map(lambda batch: {"b": batch["a"] * 2})
+print(dataset_with_duplicates.shape)
+len(dataset_with_duplicates)
+dataset_with_duplicates[:]

doc_faiss_search.py

@@ -0,0 +1,49 @@
+from datasets import load_dataset, load_from_disk, Dataset
+from transformers import AutoTokenizer, AutoModel
+import torch
+import pandas as pd
+
+model_ckpt = "nomic-ai/nomic-embed-text-v1.5"
+
+
+tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
+model = AutoModel.from_pretrained(model_ckpt, trust_remote_code=True)
+
+device = torch.device("cpu")
+model.to(device)
+
+def cls_pooling(model_output):
+    return model_output.last_hidden_state[:, 0]
+
+def get_embeddings(text_list):
+    encoded_input = tokenizer(
+        text_list, padding=True, truncation=True, return_tensors="pt"
+    )
+    encoded_input = {k: v.to(device) for k, v in encoded_input.items()}
+    model_output = model(**encoded_input)
+    return cls_pooling(model_output)
+
+
+embeddings_dataset = Dataset.load_from_disk("dataset/embeddings")
+
+embeddings_dataset.load_faiss_index("embeddings", "index/embeddings")
+
+question = "Download license key"
+
+question_embedding = get_embeddings([question]).cpu().detach().numpy()
+
+scores, samples = embeddings_dataset.get_nearest_examples(
+    "embeddings", question_embedding, k=10
+)
+
+samples_df = pd.DataFrame.from_dict(samples)
+samples_df["scores"] = scores
+samples_df.sort_values("scores", ascending=True, inplace=True)
+
+for _, row in samples_df.iterrows():
+    print(f"COMMENT: {row.text}")
+    print(f"SCORE: {row.scores}")
+    print(f"PROMPT: {row.prompt}")
+    print("=" * 50)
+    print()
+

doc_faiss_train.py

@@ -0,0 +1,105 @@
+from datasets import load_dataset, load_from_disk, Dataset
+import os
+from transformers import AutoTokenizer, AutoModel
+import torch
+import pandas as pd
+import xml.etree.ElementTree as ET 
+import glob, os
+
+rootFolder = "c:/317"
+file = open(rootFolder + "/result.csv", "w", encoding="utf-8")
+
+def parseXML(xmlFile):
+
+    prompt = xmlFile.replace("Using_", "").replace(".xml", "").replace(".", " ").replace("_", " ")
+    text  = ""
+
+    try:
+        tree = ET.parse(rootFolder + "/" + xmlFile)
+        root = tree.getroot()
+
+        for item in root.findall(".//text"):
+            text += (item.text + " ")
+
+        if len(text) > 500:
+            text = text[:500]
+
+        if text.find("а") == -1:
+            file.write(text + "\n")
+            return {"text": text, "prompt": prompt}
+        else: 
+            return None
+
+    except Exception as error:
+        print(error)
+
+
+def generator():
+    
+    for xmlFile in glob.glob("*.xml", root_dir=rootFolder):
+        print(xmlFile)
+        data = parseXML(xmlFile)
+        if not (data == None) : yield data
+
+
+ds = Dataset.from_generator(generator)
+
+file.close()
+
+##########################################################
+
+
+model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
+#model_ckpt = "sentence-transformers/multi-qa-MiniLM-L6-cos-v1"
+# model_ckpt = "sentence-transformers/msmarco-bert-base-dot-v5"
+model_ckpt = "nomic-ai/nomic-embed-text-v1.5"
+
+
+tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
+model = AutoModel.from_pretrained(model_ckpt, trust_remote_code=True)
+
+device = torch.device("cuda")
+model.to(device)
+
+def cls_pooling(model_output):
+    return model_output.last_hidden_state[:, 0]
+
+def get_embeddings(text_list):
+    encoded_input = tokenizer(
+        text_list, padding=True, truncation=True, return_tensors="pt"
+    )
+    encoded_input = {k: v.to(device) for k, v in encoded_input.items()}
+    model_output = model(**encoded_input)
+    return cls_pooling(model_output)
+
+embeddings_dataset = ds.map(
+    lambda x: {"embeddings": get_embeddings(x["text"]).detach().cpu().numpy()[0]}
+)
+
+embeddings_dataset.save_to_disk("dataset/embeddings")
+
+embeddings_dataset = Dataset.load_from_disk("dataset/embeddings")
+
+embeddings_dataset.add_faiss_index(column="embeddings")
+
+embeddings_dataset.save_faiss_index("embeddings", "index/embeddings")
+
+question = "Download license key"
+
+question_embedding = get_embeddings([question]).cpu().detach().numpy()
+
+scores, samples = embeddings_dataset.get_nearest_examples(
+    "embeddings", question_embedding, k=10
+)
+
+samples_df = pd.DataFrame.from_dict(samples)
+samples_df["scores"] = scores
+samples_df.sort_values("scores", ascending=True, inplace=True)
+
+for _, row in samples_df.iterrows():
+    print(f"COMMENT: {row.text}")
+    print(f"SCORE: {row.scores}")
+    print(f"PROMPT: {row.prompt}")
+    print("=" * 50)
+    print()
+

faiss_test.py

@@ -0,0 +1,50 @@
+from datasets import load_dataset, load_from_disk, Dataset
+import os
+from transformers import AutoTokenizer, AutoModel
+import torch
+import pandas as pd
+import faiss
+
+########################
+
+model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
+tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
+model = AutoModel.from_pretrained(model_ckpt)
+
+device = torch.device("cuda")
+model.to(device)
+
+def cls_pooling(model_output):
+    return model_output.last_hidden_state[:, 0]
+
+def get_embeddings(text_list):
+    encoded_input = tokenizer(
+        text_list, padding=True, truncation=True, return_tensors="pt"
+    )
+    encoded_input = {k: v.to(device) for k, v in encoded_input.items()}
+    model_output = model(**encoded_input)
+    return cls_pooling(model_output)
+
+embeddings_dataset = load_from_disk("dataset/embeddings")
+
+embeddings_dataset.add_faiss_index(column="embeddings")
+
+question = "How can I load a dataset offline?"
+question_embedding = get_embeddings([question]).cpu().detach().numpy()
+
+scores, samples = embeddings_dataset.get_nearest_examples(
+    "embeddings", question_embedding, k=5
+)
+
+samples_df = pd.DataFrame.from_dict(samples)
+samples_df["scores"] = scores
+samples_df.sort_values("scores", ascending=False, inplace=True)
+
+for _, row in samples_df.iterrows():
+    print(f"COMMENT: {row.comments}")
+    print(f"SCORE: {row.scores}")
+    print(f"TITLE: {row.title}")
+    print(f"URL: {row.html_url}")
+    print("=" * 50)
+    print()
+

faiss_train.py

@@ -0,0 +1,99 @@
+from datasets import load_dataset, load_from_disk, Dataset
+import os
+from transformers import AutoTokenizer, AutoModel
+import torch
+import pandas as pd
+
+datasetPath = "dataset/github.ds"
+
+if os.path.exists(datasetPath):
+    issues_dataset = load_from_disk(datasetPath)    
+else:
+    issues_dataset = load_dataset("lewtun/github-issues", split="train")
+    issues_dataset.save_to_disk(datasetPath)
+
+issues_dataset = issues_dataset.filter(
+    lambda x: (x["is_pull_request"] == False and len(x["comments"]) > 0)
+)
+
+columns = issues_dataset.column_names
+columns_to_keep = ["title", "body", "html_url", "comments"]
+columns_to_remove = set(columns_to_keep).symmetric_difference(columns)
+issues_dataset = issues_dataset.remove_columns(columns_to_remove)
+
+issues_dataset.set_format("pandas")
+df = issues_dataset[:]
+
+comments_df = df.explode("comments", ignore_index=True)
+
+comments_dataset = Dataset.from_pandas(comments_df)
+
+comments_dataset = comments_dataset.map(
+    lambda x: {"comment_length": len(x["comments"].split())}
+)
+
+comments_dataset = comments_dataset.filter(lambda x: x["comment_length"] > 15)
+
+def concatenate_text(examples):
+    return {
+        "text": examples["title"]
+        + " \n "
+        + examples["body"]
+        + " \n "
+        + examples["comments"]
+    }
+
+
+comments_dataset = comments_dataset.map(concatenate_text)
+
+########################
+
+model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
+tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
+model = AutoModel.from_pretrained(model_ckpt)
+
+device = torch.device("cuda")
+model.to(device)
+
+def cls_pooling(model_output):
+    return model_output.last_hidden_state[:, 0]
+
+def get_embeddings(text_list):
+    encoded_input = tokenizer(
+        text_list, padding=True, truncation=True, return_tensors="pt"
+    )
+    encoded_input = {k: v.to(device) for k, v in encoded_input.items()}
+    model_output = model(**encoded_input)
+    return cls_pooling(model_output)
+
+embedding = get_embeddings(comments_dataset["text"][0])
+
+embeddings_dataset = comments_dataset.map(
+    lambda x: {"embeddings": get_embeddings(x["text"]).detach().cpu().numpy()[0]}
+)
+
+embeddings_dataset.add_faiss_index(column="embeddings")
+
+# embeddings_dataset.save_to_disk("dataset/embeddings")
+
+question = "How can I load a dataset offline?"
+question_embedding = get_embeddings([question]).cpu().detach().numpy()
+
+scores, samples = embeddings_dataset.get_nearest_examples(
+    "embeddings", question_embedding, k=5
+)
+
+samples_df = pd.DataFrame.from_dict(samples)
+samples_df["scores"] = scores
+samples_df.sort_values("scores", ascending=False, inplace=True)
+
+for _, row in samples_df.iterrows():
+    print(f"COMMENT: {row.comments}")
+    print(f"SCORE: {row.scores}")
+    print(f"TITLE: {row.title}")
+    print(f"URL: {row.html_url}")
+    print("=" * 50)
+    print()
+
+
+print(issues_dataset)

requirements.txt

@@ -0,0 +1,5 @@
+datasets
+transformers
+torch
+pandas
+#faiss-gpu