File size: 7,076 Bytes
2aacaa3 123b0e8 2aacaa3 6a76dc6 2aacaa3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 |
# Import necessary libraries
import gradio as gr
import numpy as np
import pandas as pd
from rapidfuzz.distance import Levenshtein, JaroWinkler
from sentence_transformers import SentenceTransformer, util
from typing import List
import zipfile
import os
import io
def calculate_similarity(code1, code2, Ws, Wl, Wj, model_name):
model = SentenceTransformer(model_name)
embedding1 = model.encode(code1)
embedding2 = model.encode(code2)
sim_similarity = util.cos_sim(embedding1, embedding2).item()
lev_ratio = Levenshtein.normalized_similarity(code1, code2)
jaro_winkler_ratio = JaroWinkler.normalized_similarity(code1, code2)
overall_similarity = Ws * sim_similarity + Wl * lev_ratio + Wj * jaro_winkler_ratio
return "The similarity score between the two codes is: %.2f" % overall_similarity
# Define the function to process the uploaded file and return a DataFrame
def extract_and_read_compressed_file(file_path):
file_names = []
codes = []
# Handle .zip files
if file_path.endswith('.zip'):
with zipfile.ZipFile(file_path, 'r') as z:
file_names = z.namelist()
codes = [z.read(file).decode('utf-8', errors='ignore') for file in file_names]
else:
raise ValueError("Unsupported file type. Only .zip is supported.")
return file_names, codes
def filter_and_return_top(df, similarity_threshold,returned_results):
filtered_df = df[df['similarity_score'] > similarity_threshold]
return filtered_df.head(returned_results)
# Perform paraphrase mining with the specified weights
def perform_paraphrase_mining(model, codes_list, weight_semantic, weight_levenshtein, weight_jaro_winkler):
return paraphrase_mining_with_combined_score(
model,
codes_list,
weight_semantic=weight_semantic,
weight_levenshtein=weight_levenshtein,
weight_jaro_winkler=weight_jaro_winkler
)
def paraphrase_mining_with_combined_score(
model,
sentences: List[str],
show_progress_bar: bool = False,
weight_semantic: float = 1.0,
weight_levenshtein: float = 0.0,
weight_jaro_winkler: float = 0.0
):
embeddings = model.encode(
sentences, show_progress_bar=show_progress_bar, convert_to_tensor=True)
paraphrases = util.paraphrase_mining_embeddings(embeddings, score_function=util.cos_sim)
results = []
for score, i, j in paraphrases:
lev_ratio = Levenshtein.normalized_similarity(sentences[i], sentences[j])
jaro_winkler_ratio = JaroWinkler.normalized_similarity(sentences[i], sentences[j])
combined_score = (weight_semantic * score) + \
(weight_levenshtein * lev_ratio) + \
(weight_jaro_winkler * jaro_winkler_ratio)
results.append([combined_score, i, j])
results = sorted(results, key=lambda x: x[0], reverse=True)
return results
def get_sim_list(zipped_file,Ws, Wl, Wj, model_name,threshold,number_results):
file_names, codes = extract_and_read_compressed_file(zipped_file)
model = SentenceTransformer(model_name)
code_pairs = perform_paraphrase_mining(model, codes,Ws, Wl, Wj)
pairs_results = []
for score, i, j in code_pairs:
pairs_results.append({
'file_name_1': file_names[i],
'file_name_2': file_names[j],
'similarity_score': score
})
similarity_df = pd.concat([pd.DataFrame(pairs_results)], ignore_index=True)
similarity_df = similarity_df.sort_values(by='similarity_score', ascending=False)
result = filter_and_return_top(similarity_df,threshold,number_results).round(2)
return result
# Define the Gradio app
with gr.Blocks(theme=gr.themes.Glass()) as demo:
# Tab for similarity calculation
with gr.Tab("Code Pair Similarity"):
# Input components
code1 = gr.Textbox(label="Code 1")
code2 = gr.Textbox(label="Code 2")
# Accordion for weights and models
with gr.Accordion("Weights and Models", open=False):
Ws = gr.Slider(0, 1, value=0.7, label="Semantic Search Weight", step=0.1)
Wl = gr.Slider(0, 1, value=0.3, label="Levenshiern Distance Weight", step=0.1)
Wj = gr.Slider(0, 1, value=0.0, label="Jaro Winkler Weight", step=0.1)
model_dropdown = gr.Dropdown(
[("codebert", "microsoft/codebert-base"),
("graphcodebert", "microsoft/graphcodebert-base"),
("UnixCoder", "microsoft/unixcoder-base-unimodal"),
("CodeBERTa", "huggingface/CodeBERTa-small-v1"),
("CodeT5 small", "Salesforce/codet5-small"),
("PLBART", "uclanlp/plbart-java-cs"),],
label="Select Model",
value= "uclanlp/plbart-java-cs"
)
# Output component
output = gr.Textbox(label="Similarity Score")
def update_weights(Ws, Wl, Wj):
total = Ws + Wl + Wj
if total != 1:
Wj = 1 - (Ws + Wl)
return Ws, Wl, Wj
# Update weights when any slider changes
Ws.change(update_weights, [Ws, Wl, Wj], [Ws, Wl, Wj])
Wl.change(update_weights, [Ws, Wl, Wj], [Ws, Wl, Wj])
Wj.change(update_weights, [Ws, Wl, Wj], [Ws, Wl, Wj])
# Button to trigger the similarity calculation
calculate_btn = gr.Button("Calculate Similarity")
calculate_btn.click(calculate_similarity, inputs=[code1, code2, Ws, Wl, Wj, model_dropdown], outputs=output)
# Tab for file upload and DataFrame output
with gr.Tab("Code Collection Pair Similarity"):
# File uploader component
file_uploader = gr.File(label="Upload a Zip file",file_types=[".zip"])
with gr.Accordion("Weights and Models", open=False):
Ws = gr.Slider(0, 1, value=0.7, label="Semantic Search Weight", step=0.1)
Wl = gr.Slider(0, 1, value=0.3, label="Levenshiern Distance Weight", step=0.1)
Wj = gr.Slider(0, 1, value=0.0, label="Jaro Winkler Weight", step=0.1)
model_dropdown = gr.Dropdown(
[("codebert", "microsoft/codebert-base"),
("graphcodebert", "microsoft/graphcodebert-base"),
("UnixCoder", "microsoft/unixcoder-base-unimodal"),
("CodeBERTa", "huggingface/CodeBERTa-small-v1"),
("CodeT5 small", "Salesforce/codet5-small"),
("PLBART", "uclanlp/plbart-java-cs"),],
label="Select Model",
value= "uclanlp/plbart-java-cs"
)
threshold = gr.Slider(0, 1, value=0, label="Threshold", step=0.01)
number_results = gr.Slider(1, 1000, value=10, label="Number of Returned pairs", step=1)
# Output component for the DataFrame
df_output = gr.Dataframe(label="Results")
# Button to trigger the file processing
process_btn = gr.Button("Process File")
process_btn.click(get_sim_list, inputs=[file_uploader, Ws, Wl, Wj, model_dropdown,threshold,number_results], outputs=df_output)
# Launch the Gradio app with live=True
demo.launch(show_error=True,debug=True) |