import os import time from datetime import datetime import logging from pathlib import Path import requests import json import numpy as np import pandas as pd import spacy from sentence_transformers import CrossEncoder import litellm # from litellm import completion from tqdm import tqdm from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, AutoConfig, pipeline # from accelerate import PartialState # from accelerate.inference import prepare_pippy import torch import cohere from openai import OpenAI # import google import google.generativeai as genai import src.backend.util as util import src.envs as envs # import pandas as pd import scipy from scipy.spatial.distance import jensenshannon # import numpy as np # litellm.set_verbose=False litellm.set_verbose=True # Set up basic configuration for logging logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') # Load spacy model for word tokenization nlp = spacy.load("en_core_web_sm") nlp1 = spacy.load("en_core_web_trf") os.environ["HUGGINGFACE_API_KEY"] = envs.TOKEN def load_evaluation_model(model_path): """Load the evaluation model from the given path Args: model_path (str): Path to the evaluation model Returns: CrossEncoder: The evaluation model """ # model = CrossEncoder(model_path) model = "" return model class ModelLoadingException(Exception): """Exception raised for errors in loading a model. Attributes: model_id (str): The model identifier. revision (str): The model revision. """ def __init__(self, model_id, revision, messages="Error initializing model"): self.model_id = model_id self.revision = revision super().__init__(f"{messages} id={model_id} revision={revision}") class SummaryGenerator: """A class to generate summaries using a causal language model. Attributes: model (str): huggingface/{model_id} api_base (str): https://api-inference.huggingface.co/models/{model_id} summaries_df (DataFrame): DataFrame to store generated summaries. revision (str): Model revision. avg_length (float): Average length of summaries. answer_rate (float): Rate of non-empty summaries. """ def __init__(self, model_id, revision): """ Initializes the SummaryGenerator with a model. Args: model_id (str): Identifier for the model. revision (str): Revision of the model. """ self.model_id = model_id self.model = f"huggingface/{model_id}" self.api_base = f"https://api-inference.huggingface.co/models/{model_id}" self.summaries_df = pd.DataFrame() self.revision = revision self.avg_length = None self.answer_rate = None self.exceptions = None self.local_model = None def generate_summaries(self, dataset, df_prompt, save_path=None): """Generate summaries for a given DataFrame of source docs. 修改这里拉取模型生成结果 Args: df (DataFrame): DataFrame containing source docs. Returns: summaries_df (DataFrame): Generated summaries by the model. """ exceptions = [] if (save_path is not None) and os.path.exists(save_path): '''已存在文件,可以读取已经存在的测试文本''' self.summaries_df = pd.read_csv(save_path) # print(self.summaries_df['Experiment']) print(f'Loaded generated summaries from {save_path}') else: '''测试文件不存在，则需要调用指定的模型来进行测试''' # prompt = {} # for index, row in tqdm(df_prompt.iterrows(), total=df_prompt.shape[0]): # prompt['E' + row['Item']] = row['Prompt'] xls = pd.ExcelFile(dataset) sheet_names = xls.sheet_names # sheet_names = df.sheetnames print(f"Total: {len(sheet_names)}") print(sheet_names) Experiment_ID, Questions_ID, Item_ID, Condition, User_prompt, Response, Factor_2, Stimuli_1 = [], [], [], [], [] ,[], [], [] for i, sheet_name in enumerate(sheet_names, start=1): # 读取每个工作表 # if i > 2 and i ==1: # continue print(i, sheet_name) df_sheet = pd.read_excel(xls, sheet_name=sheet_name) # 假设第一列是'Prompt0'，但这里我们使用列名来避免硬编码 if 'Prompt0' in df_sheet.columns: prompt_column = df_sheet['Prompt0'] else: # 如果'Prompt0'列不存在，则跳过该工作表或进行其他处理 continue if i == 3 : word1_list = df_sheet['Stimuli-2'] word2_list = df_sheet['Stimuli-3'] V2_column = [] for jj in range(len(word1_list)): V2_column.append(word1_list[jj] + '_' + word2_list[jj]) # print(V2_column) elif i == 9: V2_column = df_sheet['V2'] #SL, LS elif i == 4 or i == 6 : V2_column = df_sheet['Stimuli-2'] #Stimuli-2 else: V2_column = [""] * len(prompt_column) q_column = df_sheet["ID"] Item_column = df_sheet["Item"] Condition_column = df_sheet["Condition"] Stimuli_1_column = df_sheet["Stimuli-1"] if 'Stimuli-2' in df_sheet.columns: Stimuli_2_column = df_sheet["Stimuli-2"] # 遍历Prompt0列的值 for j, prompt_value in enumerate(tqdm(prompt_column, desc=f"Processing {sheet_name}"), start=0): ID = 'E' + str(i) # q_ID = ID + '_' + str(j) # print(ID, q_ID, prompt_value) system_prompt = envs.SYSTEM_PROMPT _user_prompt = prompt_value for ii in range(10): # user_prompt = f"{envs.USER_PROMPT}\nPassage:\n{_source}" while True: try: '''调用''' print('开始调用LLM-API') _response = self.generate_summary(system_prompt, _user_prompt) # print(f"Finish index {index}") break except Exception as e: if 'Rate limit reached' in str(e): wait_time = 3660 current_time = datetime.now().strftime('%H:%M:%S') print(f"Rate limit hit at {current_time}. Waiting for 1 hour before retrying...") time.sleep(wait_time) elif 'is currently loading' in str(e): wait_time = 200 print(f"Model is loading, wait for {wait_time}") time.sleep(wait_time) elif '429 Resource has been exhausted' in str(e): # for gemini models wait_time = 60 print(f"Quota has reached, wait for {wait_time}") time.sleep(wait_time) else: print(f"Error at index {i}: {e}") _response = "" exceptions.append(i) break if i == 5: print(_response) if _response == None: _response1, _response2 = "", "" else: try: import re _response1,_response2 = re.split(r'\n\s*\n', _response.strip()) except: _response1 = _response.split('\n\n') if len(_response) == 2: _response1, _response2 = _response[0], _response[1] else: _response1, _response2 = _response[0], "" Experiment_ID.append(ID) Questions_ID.append(q_column[j]) User_prompt.append(_user_prompt) Response.append(_response2) Factor_2.append(V2_column[j]) Stimuli_1.append(Stimuli_2_column[j]) Item_ID.append(Item_column[j]) Condition.append(Condition_column[j]) # the first sentence in the response is saved as E51 Experiment_ID.append(ID + '1') Questions_ID.append(str(q_column[j]) + '1') User_prompt.append(_user_prompt) Response.append(_response1) Factor_2.append(V2_column[j]) Stimuli_1.append(Stimuli_1_column[j]) Item_ID.append(Item_column[j]) Condition.append(Condition_column[j]) else: Experiment_ID.append(ID) Questions_ID.append(q_column[j]) User_prompt.append(_user_prompt) Response.append(_response) if i == 6: Factor_2.append(Condition_column[j]) Stimuli_1.append(V2_column[j]) else: Factor_2.append(V2_column[j]) Stimuli_1.append(Stimuli_1_column[j]) Item_ID.append(Item_column[j]) Condition.append(Condition_column[j]) print(_response) # exit() # Sleep to prevent hitting rate limits too frequently time.sleep(1) self.summaries_df = pd.DataFrame(list(zip(Experiment_ID, Questions_ID, Item_ID, Condition, User_prompt, Response, Factor_2, Stimuli_1)), columns=["Experiment", "Question_ID", "Item", "Condition", "User_prompt", "Response","Factor 2","Stimuli 1"]) if save_path is not None: print(f'Save summaries to {save_path}') fpath = Path(save_path) fpath.parent.mkdir(parents=True, exist_ok=True) self.summaries_df.to_csv(fpath) self.exceptions = exceptions # self._compute_avg_length() # self._compute_answer_rate() return self.summaries_df def generate_summary(self, system_prompt: str, user_prompt: str): # Using Together AI API using_together_api = False together_ai_api_models = ['mixtral', 'dbrx', 'wizardlm'] for together_ai_api_model in together_ai_api_models: if together_ai_api_model in self.model_id.lower(): using_together_api = True break # print('适用哪一种LLM',together_ai_api_model , using_together_api) # print(self.model_id.lower()) #meta-llama/llama-2-7b-chat-hf # print('local',self.local_model) $None # exit() # if 'mixtral' in self.model_id.lower() or 'dbrx' in self.model_id.lower() or 'wizardlm' in self.model_id.lower(): # For mixtral and dbrx models, use Together AI API if using_together_api: # suffix = "completions" if ('mixtral' in self.model_id.lower() or 'base' in self.model_id.lower()) else "chat/completions" suffix = "chat/completions" url = f"https://api.together.xyz/v1/{suffix}" payload = { "model": self.model_id, # "max_tokens": 4096, 'max_new_tokens': 50, # "temperature": 0.0, # 'repetition_penalty': 1.1 if 'mixtral' in self.model_id.lower() else 1 } # if 'mixtral' in self.model_id.lower(): # # payload['prompt'] = user_prompt # # payload['prompt'] = "Write a summary of the following passage:\nPassage:\n" + user_prompt.split('Passage:\n')[-1] + '\n\nSummary:' # payload['prompt'] = 'You must stick to the passage provided. Provide a concise summary of the following passage, covering the core pieces of information described:\nPassage:\n' + user_prompt.split('Passage:\n')[-1] + '\n\nSummary:' # print(payload) # else: # payload['messages'] = [{"role": "system", "content": system_prompt}, # {"role": "user", "content": user_prompt}] payload['messages'] = [{"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}] headers = { "accept": "application/json", "content-type": "application/json", "Authorization": f"Bearer {os.environ['TOGETHER_API_KEY']}" } response = requests.post(url, json=payload, headers=headers) try: result = json.loads(response.text) # print(result) result = result["choices"][0] if 'message' in result: result = result["message"]["content"].strip() else: result = result["text"] result_candidates = [result_cancdidate for result_cancdidate in result.split('\n\n') if len(result_cancdidate) > 0] result = result_candidates[0] print(result) except: print(response) result = '' print(result) return result # Using OpenAI API elif 'gpt' in self.model_id.lower(): response = litellm.completion( model=self.model_id.replace('openai/',''), messages=[{"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}], # temperature=0.0, max_tokens=50, ) result = response['choices'][0]['message']['content'] # print() print(result) return result # Using Google AI API for Gemini models elif 'gemini' in self.model_id.lower(): genai.configure(api_key=os.getenv('GOOGLE_AI_API_KEY')) generation_config = { "temperature": 0, "top_p": 0.95, # cannot change "top_k": 0, "max_output_tokens": 50, # "response_mime_type": "application/json", } safety_settings = [ { "category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE" }, { "category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE" }, { "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE" }, { "category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE" }, ] model = genai.GenerativeModel(model_name="gemini-1.5-pro-latest" if "gemini-1.5-pro" in self.model_id.lower() else self.model_id.lower().split('google/')[-1], generation_config=generation_config, system_instruction=system_prompt, safety_settings=safety_settings) convo = model.start_chat(history=[]) convo.send_message(user_prompt) # print(convo.last) result = convo.last.text print(result) return result # Using HF API or download checkpoints elif self.local_model is None: # print(self.model_id) # print(self.api_base) # mistralai/Mistral-7B-Instruct-v0.1 # https://api-inference.huggingface.co/models/mistralai/Mistral-7B-Instruct-v0.1 try: # try use HuggingFace API # response = litellm.completion( # model="huggingface/"+'command-r-plus' if 'command' in self.model_id else self.model_id, # messages=[{"role": "system", "content": system_prompt}, # {"role": "user", "content": user_prompt}], # temperature=0.0, # max_tokens=1024, # api_base= "https://api-inference.huggingface.co/models/" + self.model_id, # ) self.model_id = 'command-r-plus' if 'command' in self.model_id else self.model_id response = litellm.completion( model="huggingface/" + self.model_id, # mistralai/Mistral-7B-Instruct-v0.1", messages=[{"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}], #temperature=0.0, max_tokens=1024, api_base="https://api-inference.huggingface.co/models/" + self.model_id) print("模型返回结果",response) print("模型返回结果结束") # exit() result = response['choices'][0]['message']['content'] print(result) return result # exit() except: # fail to call api. run it locally. self.tokenizer = AutoTokenizer.from_pretrained(self.model_id, trust_remote_code=True) print("Tokenizer loaded") self.local_model = AutoModelForCausalLM.from_pretrained(self.model_id, trust_remote_code=True, device_map="auto", torch_dtype="auto", cache_dir='/home/paperspace/cache') print("Local model loaded") # exit() # Using local model if self.local_model: # cannot call API. using local model messages=[ {"role": "system", "content": system_prompt}, # gemma-1.1 does not accept system role {"role": "user", "content": user_prompt} ] try: # some models support pipeline pipe = pipeline( "text-generation", model=self.local_model, tokenizer=self.tokenizer, ) generation_args = { "max_new_tokens": 50, "return_full_text": False, #"temperature": 0.0, "do_sample": False, } output = pipe(messages, **generation_args) result = output[0]['generated_text'] print(result) except: prompt = self.tokenizer.apply_chat_template(messages,add_generation_prompt=True, tokenize=False) print(prompt) input_ids = self.tokenizer(prompt, return_tensors="pt").to('cuda') with torch.no_grad(): outputs = self.local_model.generate(**input_ids, max_new_tokens=50, do_sample=True, pad_token_id=self.tokenizer.eos_token_id) result = self.tokenizer.decode(outputs[0], skip_special_tokens=True) result = result.replace(prompt[0], '') print(result) return result def _compute_avg_length(self): """ Compute the average length of non-empty summaries using SpaCy. """ total_word_count = 0 total_count = 0 for summary in self.summaries_df['summary']: if util.is_summary_valid(summary): doc = nlp(summary) words = [token.text for token in doc if token.is_alpha] total_word_count += len(words) total_count += 1 self.avg_length = 0 if total_count == 0 else total_word_count / total_count def _compute_answer_rate(self): """ Compute the rate of non-empty summaries. """ valid_count = sum(1 for summary in self.summaries_df['summary'] if util.is_summary_valid(summary)) total_count = len(self.summaries_df) self.answer_rate = 0 if total_count == 0 else valid_count / total_count class EvaluationModel: """A class to evaluate generated summaries. Attributes: model (CrossEncoder): The evaluation model. scores (list): List of evaluation scores. accuracy (float): Accuracy of the summaries. hallucination_rate (float): Rate of hallucination in summaries. """ def __init__(self, model_path): """ Initializes the EvaluationModel with a CrossEncoder model. Args: model_path (str): Path to the CrossEncoder model. """ self.model = load_evaluation_model(model_path) self.scores = [] self.factual_consistency_rate = None self.hallucination_rate = None self.humanlike_score = None def code_results(self, summaries_df): '''code results from LLM's response''' output = [] '''database for Exp4''' item4 = pd.read_csv(envs.ITEM_4_DATA) wordpair2code = {} for j in range(len(item4['Coding'])): wordpair2code[item4['Pair'][j]] = item4['Coding'][j] '''verb for Exp5''' item5 = pd.read_csv(envs.ITEM_5_DATA) # item corresponding to verb, same item id corresponding to verb pair item2verb2 = {} item2verb1 = {} Stimuli1, Stimuli2 = {}, {} for j in range(len(item5['Item'])): item2verb1[item5['Item'][j]] = item5['Verb1'][j] item2verb2[item5['Item'][j]] = item5['Verb2'][j] Stimuli1[item5['ID'][j]] = item5['Stimuli-1'][j] Stimuli2[item5['ID'][j]] = item5['Stimuli-2'][j] male_keyword = ["he", "his", "himself"] female_keyword = ["she", "her", "herself"] print(len(summaries_df["Experiment"])) for i in range(len(summaries_df["Experiment"])): # vote_1_1, vote_1_2, vote_1_3 = 0, 0, 0 # print() if pd.isna(summaries_df["Response"][i]): output.append("Other") continue rs = summaries_df["Response"][i].strip().lower() '''Exp1''' if summaries_df["Experiment"][i] == "E1": print("E1", rs) rs = rs.replace('"','') if rs == "round": # vote_1_1 += 1 output.append("Round") elif rs == "spiky": output.append("Spiky") else: output.append("Other") '''Exp2''' elif summaries_df["Experiment"][i] == "E2": # rs = summaries_df["Response"][i].strip() rs = rs.split(' ') print("E2", rs) male, female = 0, 0 for word in rs: if word in female_keyword and male == 0: female = 1 output.append("Female") break if word in male_keyword and female == 0: male = 1 output.append("Male") break if male == 0 and female == 0 : output.append("Other") '''Exp3''' elif summaries_df["Experiment"][i] == "E3": # rs = summaries_df["Response"][i].strip() print("E3", rs) if pd.isna(summaries_df["Factor 2"][i]): output.append("Other") else: if summaries_df["Factor 2"][i].strip() == "LS": if "2" in rs: output.append("Long") elif "3" in rs: output.append("Short") else: output.append("Other") if summaries_df["Factor 2"][i].strip() == "SL": if "2" in rs: output.append("Short") elif "3" in rs: output.append("Long") else: output.append("Other") '''Exp4''' elif summaries_df["Experiment"][i] == "E4": # rs = summaries_df["Response"][i].strip() target = summaries_df["Factor 2"][i].strip().lower() pair = target + "_" + rs print("E4:", pair) if pair in wordpair2code.keys(): output.append(wordpair2code[pair]) else: output.append("Other") '''Exp5''' elif summaries_df["Experiment"][i] == "E5" or summaries_df["Experiment"][i] == "E51": # sentence = summaries_df["Response"][i].strip() item_id = summaries_df["Item"][i] question_id = summaries_df["Question_ID"][i] sti1, sti2 = "", "" if summaries_df["Experiment"][i] == "E51": sti1 = Stimuli1[question_id[0:-1]].lower().replace("...", "") sti2 = Stimuli2[question_id[0:-1]].lower().replace("...", "") verb = item2verb1[item_id].lower() sentence = sti1 + " " + rs.replace(sti1, "") print("E5", verb, sentence) if summaries_df["Experiment"][i] == "E5": sti1 = Stimuli1[question_id].lower().replace("...", "") # print(sti1) sti2 = Stimuli2[question_id].lower().replace("...", "") verb = item2verb2[item_id].lower() sentence = sti2.replace("...","") + " " + rs.replace(sti2, "") print("E5", verb, sentence) doc = nlp1(sentence.replace(" "," ")) # print(doc) # print() verb_token = None for token in doc: # print(token.lemma_) if token.lemma_ == verb: verb_token = token break # exit() if verb_token is None: output.append("Other") print("E5 The target verb is missing from the sentence.") else: pobj, dative = None, None # print(verb_token.children) # exit() for child in verb_token.children: print(child) if (child.dep_ == 'dative' and child.pos_ == "ADP") or (child.text == "to" and child.dep_ == 'prep' and child.pos_ == "ADP"): pobj = child.text if child.dep_ == 'dative': dative = child.text print("E5", pobj, dative) # exit() if pobj: output.append("PO") elif dative: output.append("DO") else: print("Other", sentence, pobj, dative) # exit() output.append("Other") '''Exp6''' elif summaries_df["Experiment"][i] == "E6": sentence = summaries_df["Stimuli 1"][i].strip().lower() print("E6", sentence) doc = nlp1(sentence) subject = "None" obj = "None" # 遍历依存关系，寻找主语和宾语 for token in doc: if token.dep_ == "nsubj": subject = token.text elif token.dep_ == "dobj": obj = token.text print("E6", subject, obj) if subject in rs and obj in rs: print(rs, subject, obj, "Other") output.append("Other") elif subject in rs: print(rs, subject, obj, "VP") output.append("VP") elif obj in rs: print(rs, subject, obj, "NP") output.append("NP") else: print(rs, subject, obj, "Other") output.append("Other") '''Exp7''' elif summaries_df["Experiment"][i] == "E7": # rs = summaries_df["Response"][i].strip().lower() print("E7",rs) if rs == "no": output.append("0") elif rs == "yes": output.append("1") else: output.append("Other") '''Exp8''' elif summaries_df["Experiment"][i] == "E8": # rs = summaries_df["Response"][i].strip() if "something is wrong with the question" in rs: output.append("1") else: output.append("0") '''Exp9''' elif summaries_df["Experiment"][i] == "E9": male, female = 0, 0 # rs = summaries_df["Response"][i].strip() if "because" in rs: rs = rs.replace("because because","because").split("because")[1] else: rs = rs condition = summaries_df["Factor 2"][i].strip() rs = rs.split(" ") for w in rs: if w in male_keyword and female != 1: male = 1 break if w in female_keyword and male != 1: female = 1 break print("E9", "condition", condition, "male", male, "female", female) if male == 0 and female == 0: output.append('Other') else: if male == 1 and female==0: if condition == "MF": output.append("Subject") elif condition == "FM": output.append("Object") else: output.append("Other") elif female == 1 and male ==0: if condition == "MF": output.append("Object") elif condition == "FM": output.append("Subject") else: output.append("Other") '''Exp10''' elif summaries_df["Experiment"][i] == "E10": # rs = summaries_df["Response"][i].strip() if rs == "yes": output.append("1") else: output.append("0") else: print("can;t find the Exp:", summaries_df["Experiment"][i]) output.append("NA") # print(output) # exit() '''human''' self.data = pd.DataFrame(list(zip(summaries_df["Experiment"], summaries_df["Question_ID"], summaries_df["Item"], summaries_df["Response"], summaries_df["Factor 2"], summaries_df["Stimuli 1"], summaries_df["Coding"], output)), columns=["Experiment", "Question_ID", "Item", "Response", "Factor 2", "Simulate 1","Original_Coding","Coding"]) # '''LLM''' # self.data = pd.DataFrame(list(zip(summaries_df["Experiment"], summaries_df["Question_ID"], summaries_df["Item"], summaries_df["Response"], summaries_df["Factor 2"], summaries_df["Stimuli 1"], output)), # columns=["Experiment", "Question_ID", "Item", "Response", "Factor 2", "Simulate 1","Coding"]) print(self.data.head()) return self.data def code_results_llm(self, summaries_df): '''code results from LLM's response''' output = [] '''database for Exp4''' item4 = pd.read_csv(envs.ITEM_4_DATA) wordpair2code = {} for j in range(len(item4['Coding'])): wordpair2code[item4['Pair'][j]] = item4['Coding'][j] '''verb for Exp5''' item5 = pd.read_csv(envs.ITEM_5_DATA) # item corresponding to verb, same item id corresponding to verb pair item2verb2 = {} item2verb1 = {} Stimuli1, Stimuli2 = {}, {} for j in range(len(item5['Item'])): item2verb1[item5['Item'][j]] = item5['Verb1'][j] item2verb2[item5['Item'][j]] = item5['Verb2'][j] Stimuli1[item5['ID'][j]] = item5['Stimuli-1'][j] Stimuli2[item5['ID'][j]] = item5['Stimuli-2'][j] male_keyword = ["he", "his", "himself"] female_keyword = ["she", "her", "herself"] print(len(summaries_df["Experiment"])) for i in range(len(summaries_df["Experiment"])): # vote_1_1, vote_1_2, vote_1_3 = 0, 0, 0 # print() if pd.isna(summaries_df["Response"][i]): output.append("Other") continue rs = summaries_df["Response"][i].strip().lower() '''Exp1''' if summaries_df["Experiment"][i] == "E1": print("E1", rs) rs = rs.replace('"','') if rs == "round": # vote_1_1 += 1 output.append("Round") elif rs == "spiky": output.append("Spiky") else: output.append("Other") '''Exp2''' elif summaries_df["Experiment"][i] == "E2": # rs = summaries_df["Response"][i].strip() rs = rs.split(' ') print("E2", rs) male, female = 0, 0 for word in rs: if word in female_keyword and male == 0: female = 1 output.append("Female") break if word in male_keyword and female == 0: male = 1 output.append("Male") break if male == 0 and female == 0 : output.append("Other") '''Exp3''' elif summaries_df["Experiment"][i] == "E3": # rs = summaries_df["Response"][i].strip() print("E3", rs) rs = rs.replace('"', '') pair = summaries_df["Factor 2"][i] word1, word2 = pair.split('_') if rs == word1: if len(word1) > len(word2): output.append("Long") else: output.append("Short") elif rs == word2: if len(word1) > len(word2): output.append("Short") else: output.append("Long") else: output.append("Other") '''Exp4''' elif summaries_df["Experiment"][i] == "E4": # rs = summaries_df["Response"][i].strip() meaning_word = rs.split(";")[4].replace(" ",'') target = summaries_df["Factor 2"][i].strip().lower() pair = target + "_" + meaning_word print("E4:", pair) if pair in wordpair2code.keys(): output.append(wordpair2code[pair]) else: output.append("Other") '''Exp5''' elif summaries_df["Experiment"][i] == "E5" or summaries_df["Experiment"][i] == "E51": # sentence = summaries_df["Response"][i].strip() item_id = summaries_df["Item"][i] question_id = summaries_df["Question_ID"][i] sti1, sti2 = "", "" if summaries_df["Experiment"][i] == "E51": sti1 = Stimuli1[question_id[0:-1]].lower().replace("...", "") sti2 = Stimuli2[question_id[0:-1]].lower().replace("...", "") verb = item2verb1[item_id].lower() sentence = sti1 + " " + rs.replace(sti1, "") print("E5", verb, sentence) if summaries_df["Experiment"][i] == "E5": sti1 = Stimuli1[question_id].lower().replace("...", "") # print(sti1) sti2 = Stimuli2[question_id].lower().replace("...", "") verb = item2verb2[item_id].lower() sentence = sti2.replace("...","") + " " + rs.replace(sti2, "") print("E5", verb, sentence) doc = nlp1(sentence.replace(" "," ")) # print(doc) # print() verb_token = None for token in doc: # print(token.lemma_) if token.lemma_ == verb: verb_token = token break # exit() if verb_token is None: output.append("Other") print("E5 The target verb is missing from the sentence.") else: pobj, dative = None, None # print(verb_token.children) # exit() for child in verb_token.children: print(child) if (child.dep_ == 'dative' and child.pos_ == "ADP") or (child.text == "to" and child.dep_ == 'prep' and child.pos_ == "ADP"): pobj = child.text if child.dep_ == 'dative': dative = child.text print("E5", pobj, dative) # exit() if pobj: output.append("PO") elif dative: output.append("DO") else: print("Other", sentence, pobj, dative) # exit() output.append("Other") '''Exp6''' elif summaries_df["Experiment"][i] == "E6": sentence = summaries_df["Stimuli 1"][i].strip().lower() print("E6", sentence) doc = nlp1(sentence) subject = "None" obj = "None" # 遍历依存关系，寻找主语和宾语 for token in doc: if token.dep_ == "nsubj": subject = token.text elif token.dep_ == "dobj": obj = token.text print("E6", subject, obj) if subject in rs and obj in rs: print(rs, subject, obj, "Other") output.append("Other") elif subject in rs: print(rs, subject, obj, "VP") output.append("VP") elif obj in rs: print(rs, subject, obj, "NP") output.append("NP") else: print(rs, subject, obj, "Other") output.append("Other") '''Exp7''' elif summaries_df["Experiment"][i] == "E7": # rs = summaries_df["Response"][i].strip().lower() rs = rs.replace(".", "").replace(",", "") print("E7",rs) if rs == "no": output.append("0") elif rs == "yes": output.append("1") else: output.append("Other") '''Exp8''' elif summaries_df["Experiment"][i] == "E8": # rs = summaries_df["Response"][i].strip() print("E8",rs) if "something is wrong with the question" in rs: output.append("1") else: output.append("0") '''Exp9''' elif summaries_df["Experiment"][i] == "E9": male, female = 0, 0 # rs = summaries_df["Response"][i].strip() if "because" in rs: rs = rs.replace("because because","because").split("because")[1] else: rs = rs condition = summaries_df["Factor 2"][i].strip() rs = rs.split(" ") for w in rs: if w in male_keyword and female != 1: male = 1 break if w in female_keyword and male != 1: female = 1 break print("E9", "condition", condition, "male", male, "female", female) if male == 0 and female == 0: output.append('Other') else: if male == 1 and female==0: if condition == "MF": output.append("Subject") elif condition == "FM": output.append("Object") else: output.append("Other") elif female == 1 and male ==0: if condition == "MF": output.append("Object") elif condition == "FM": output.append("Subject") else: output.append("Other") '''Exp10''' elif summaries_df["Experiment"][i] == "E10": # rs = summaries_df["Response"][i].strip() rs = rs.replace(".", "") if rs == "yes": output.append("1") else: output.append("0") else: print("can;t find the Exp:", summaries_df["Experiment"][i]) output.append("NA") # print(output) # exit() '''human''' # self.data = pd.DataFrame(list(zip(summaries_df["Experiment"], summaries_df["Question_ID"], summaries_df["Item"], summaries_df["Response"], summaries_df["Factor 2"], summaries_df["Stimuli 1"], summaries_df["Coding"], output)), # columns=["Experiment", "Question_ID", "Item", "Response", "Factor 2", "Simulate 1","Original_Coding","Coding"]) '''LLM''' self.data = pd.DataFrame(list(zip(summaries_df["Experiment"], summaries_df["Question_ID"], summaries_df["Item"], summaries_df["Response"], summaries_df["Factor 2"], summaries_df["Stimuli 1"], output)), columns=["Experiment", "Question_ID", "Item", "Response", "Factor 2", "Simulate 1","Coding"]) print(self.data.head()) return self.data def calculate_js_divergence(self, file_path_1, file_path_2): """ Calculate the Jensen-Shannon divergence for response distributions between two datasets. - Extracts E5 and E51 pairs, creates new data based on comparison, removes the original E5 and E51, and then calculates the JS divergence between the datasets. Parameters: file_path_1 (str): Path to the first dataset file (Excel format). file_path_2 (str): Path to the second dataset file (CSV format). Returns: float: The average JS divergence across all common Question_IDs. """ # Load the datasets human_df = pd.read_excel(file_path_1) llm_df = pd.read_csv(file_path_2) def create_e5_entries(df): new_entries = [] for i in range(len(df) - 1): if 'E51' in df.iloc[i]['Experiment']: priming_id = df.iloc[i][0]-1 priming_row_id = df[df.iloc[:, 0] == priming_id].index[0] new_question_id = df.iloc[priming_row_id]['Question_ID'] label = 1 if df.iloc[i]['Coding'] == df.iloc[priming_row_id]['Coding'] else 0 new_entries.append({ 'Question_ID': new_question_id, 'Response': f'{df.iloc[i]["Coding"]}-{df.iloc[priming_row_id]["Coding"]}', 'Coding': label }) return pd.DataFrame(new_entries) # Create new E5 entries for both datasets human_e5 = create_e5_entries(human_df) llm_e5 = create_e5_entries(llm_df) # Remove E5 and E51 entries from both datasets human_df = human_df[~human_df['Question_ID'].str.contains('E5')] llm_df = llm_df[~llm_df['Question_ID'].str.contains('E5')] # Append new E5 entries to the cleaned dataframes human_df = pd.concat([human_df, human_e5], ignore_index=True) llm_df = pd.concat([llm_df, llm_e5], ignore_index=True) ### Calculate Average JS Divergence ### # Extract the relevant columns for JS divergence calculation human_responses = human_df[['Question_ID', 'Coding']] llm_responses = llm_df[['Question_ID', 'Coding']] # Get unique Question_IDs present in both datasets common_question_ids = set(human_responses['Question_ID']).intersection(set(llm_responses['Question_ID'])) # Initialize a list to store JS divergence for each Question_ID js_divergence_list = [] js_divergence ={} # Calculate JS divergence for each common Question_ID for q_id in common_question_ids: # Get response distributions for the current Question_ID in both datasets human_dist = human_responses[human_responses['Question_ID'] == q_id]['Coding'].value_counts(normalize=True) llm_dist = llm_responses[llm_responses['Question_ID'] == q_id]['Coding'].value_counts(normalize=True) # Reindex the distributions to have the same index, filling missing values with 0 all_responses = set(human_dist.index).union(set(llm_dist.index)) human_dist = human_dist.reindex(all_responses, fill_value=0) llm_dist = llm_dist.reindex(all_responses, fill_value=0) # Calculate JS divergence and add to the list js_div = jensenshannon(human_dist, llm_dist, base=2) experiment_id = q_id.split('_')[1] if experiment_id not in js_divergence: js_divergence[experiment_id] = [] js_divergence[experiment_id].append(js_div) js_divergence_list.append(js_div) #js_divergence[q_id] = js_div # Calculate the average JS divergence # JS per experiment avg_js_divergence_per_experiment = {exp: 1- np.nanmean(divs) for exp, divs in js_divergence.items()} print(avg_js_divergence_per_experiment) # JS overall avg_js_divergence = 1 - np.nanmean(js_divergence_list) print("avg_js_divergence:", avg_js_divergence) return avg_js_divergence def evaluate_humanlike(self, summaries_df, human_data_path, result_save_path): ''' evaluate humanlike score 1. code the result 2. comput the similaritirs between human and model process model responses''' '''coding human data''' # self.huamn_df = pd.read_csv(human_data_path) # self.data = self.code_results(self.huamn_df) save_path = human_data_path.replace('.csv','_coding.csv') human_save_path = "./src/datasets/coding_human.xlsx" # if save_path is not None: # print(f'Save human coding results to {save_path}') # fpath = Path(save_path) # fpath.parent.mkdir(parents=True, exist_ok=True) # self.data.to_csv(fpath) '''coding llm data''' save_path = result_save_path.replace('.csv','_coding.csv') self.llm_df = self.code_results_llm(summaries_df) if save_path is not None: print(f'Save LLM coding results to {save_path}') fpath = Path(save_path) fpath.parent.mkdir(parents=True, exist_ok=True) self.llm_df.to_csv(fpath) # file_path_1 = '/Users/simon/Downloads/coding_human.xlsx' # file_path_2 = '/Users/simon/Downloads/Meta-Llama-3.1-70B-Instruct_coding.csv' avg_js_divergence = self.calculate_js_divergence("./src/datasets/coding_human.xlsx", save_path) return avg_js_divergence def evaluate_hallucination(self, summaries_df): """ Evaluate the hallucination rate in summaries. Updates the 'scores' attribute of the instance with the computed scores. Args: summaries_df (DataFrame): DataFrame containing source docs and summaries. Returns: list: List of hallucination scores. Also updates the 'scores' attribute of the instance. """ hem_scores = [] sources = [] summaries = [] source_summary_pairs = util.create_pairs(summaries_df) '''评价模型结果''' for doc, summary in tqdm(source_summary_pairs, desc="Evaluating Humanlikeness"): if util.is_summary_valid(summary): try: summary = summary.replace('','').replace('','') score = self.model.predict([doc, summary])# [0] if not isinstance(score, float): try: score = score.item() except: logging.warning(f"Score type mismatch: Expected float, got {type(score)}.") continue hem_scores.append(score) sources.append(doc) summaries.append(summary) except Exception as e: logging.error(f"Error while running HEM: {e}") raise self.scores = hem_scores eval_results = {'source': sources, 'summary': summaries, 'HEM scores': hem_scores} return hem_scores, eval_results # for doc, summary in tqdm(source_summary_pairs, desc="Evaluating hallucinations"): # if util.is_summary_valid(summary): # try: # # summary_pieces = summary.split('\n') # # summary = summary_pieces[0] if len(summary_pieces[0].strip()) > 0 else summary_pieces[1] # summary = summary.replace('','').replace('','') # # print([doc, summary]) # # print(self.model.predict([doc, summary])) # score = self.model.predict([doc, summary])# [0] # if not isinstance(score, float): # try: # score = score.item() # except: # logging.warning(f"Score type mismatch: Expected float, got {type(score)}.") # continue # hem_scores.append(score) # sources.append(doc) # summaries.append(summary) # except Exception as e: # logging.error(f"Error while running HEM: {e}") # raise # self.scores = hem_scores # eval_results = {'source': sources, 'summary': summaries, 'HEM scores': hem_scores} # return hem_scores, eval_results def compute_factual_consistency_rate(self, threshold=0.5): """ Compute the factual consistency rate of the evaluated summaries based on the previously calculated scores. This method relies on the 'scores' attribute being populated, typically via the 'evaluate_hallucination' method. Returns: float: Factual Consistency Rate. Also updates the 'factual_consistency_rate' and 'hallucination_rate' attributes of the instance. Raises: ValueError: If scores have not been calculated prior to calling this method. """ if not self.scores: error_msg = "Scores not calculated. Call evaluate_hallucination() first." logging.error(error_msg) raise ValueError(error_msg) # Use threshold of 0.5 to compute factual_consistency_rate num_above_threshold = sum(score >= threshold for score in self.scores) num_total = len(self.scores) if not num_total: raise ValueError("No scores available to compute factual consistency rate.") self.factual_consistency_rate = (num_above_threshold / num_total) * 100 self.hallucination_rate = 100 - self.factual_consistency_rate return self.factual_consistency_rate