function/chatbot.py

from langchain.text_splitter import CharacterTextSplitter
import json
import os
import random
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
import google.generativeai as genai
import nltk
import pandas as pd
from groq import Groq
from langchain.chains.summarize import load_summarize_chain
from langchain.docstore.document import Document
from langchain.prompts import PromptTemplate
from langchain.retrievers import BM25Retriever, EnsembleRetriever
from langchain.retrievers.contextual_compression import ContextualCompressionRetriever
from langchain.text_splitter import CharacterTextSplitter
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_cohere import CohereRerank
from langchain_community.document_loaders import Docx2txtLoader
from langchain_community.document_loaders import TextLoader
from langchain_community.document_loaders import UnstructuredCSVLoader
from langchain_community.document_loaders import UnstructuredExcelLoader
from langchain_community.document_loaders import UnstructuredHTMLLoader
from langchain_community.document_loaders import UnstructuredMarkdownLoader
from langchain_community.document_loaders import UnstructuredPDFLoader
from langchain_community.document_loaders import UnstructuredPowerPointLoader
from langchain_community.document_loaders import UnstructuredXMLLoader
from langchain_community.document_loaders.csv_loader import CSVLoader
from langchain_community.llms import Cohere
from langchain_community.vectorstores import Chroma
from langchain_core.output_parsers.openai_tools import PydanticToolsParser
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.pydantic_v1 import BaseModel, Field
from langchain_core.runnables import RunnablePassthrough
from langchain_openai import ChatOpenAI
from typing import List
nltk.download('punkt')

def process_json_file(file_path):
    json_data = []
    with open(file_path, 'r') as file:
        for line in file:
            try:
                data = json.loads(line)
                json_data.append(data)
            except json.JSONDecodeError:
                try:
                    data = json.loads(line[:-1])
                    json_data.append(data)
                except json.JSONDecodeError as e:
                    print(f"Error decoding JSON: {e}")
    return json_data


from dotenv import load_dotenv
import os
load_dotenv()
GROQ_API_KEY = os.getenv("GROQ_API_KEY")
COHERE_API_KEY = os.getenv("COHERE_API_KEY")
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
GOOGLE_API_KEY1= os.getenv("GOOGLE_API_KEY_1")
GOOGLE_API_KEY= os.getenv("GOOGLE_API_KEY")
os.environ["COHERE_API_KEY"] = COHERE_API_KEY
os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
client = Groq(
    api_key= GROQ_API_KEY,
)
genai.configure(api_key=GOOGLE_API_KEY1)
os.environ["GOOGLE_API_KEY"] = GOOGLE_API_KEY
from langchain_google_genai import GoogleGenerativeAIEmbeddings, ChatGoogleGenerativeAI
embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
llm = ChatGoogleGenerativeAI(model='gemini-pro',
                             max_output_tokens=2048,
                             temperature=0.3,
                             convert_system_message_to_human=True)
def extract_multi_metadata_content(texts, tests):
    extracted_content = []
    precomputed_metadata = [x.metadata['source'].lower() for x in texts]
    for idx, test in enumerate(tests):
        temp_content = []
        test_terms = set(test.lower().split())  # Use set for faster lookup
        for metadata_lower, x in zip(precomputed_metadata, texts):
            if any(term in metadata_lower for term in test_terms):
                temp_content.append(x.page_content)
        if idx == 0:
            extracted_content.append(f"Dữ liệu của {test}:\n{''.join(temp_content)}")
        else:
            extracted_content.append(''.join(temp_content))
    return '\n'.join(extracted_content)

def find_matching_files_in_docs_12_id(text, id):
    folder_path = f"./user_file/{id}"
    search_terms = []
    search_terms_old = []
    matching_index = []
    search_origin = re.findall(r'\b\w+\.\w+\b|\b\w+\b', text)
    search_terms_origin = []
    for word in search_origin:
        if '.' in word:
            search_terms_origin.append(word)
        else:
            search_terms_origin.extend(re.findall(r'\b\w+\b', word))

    file_names_with_extension = re.findall(r'\b\w+\.\w+\b|\b\w+\b', text.lower())
    file_names_with_extension_old = re.findall(r'\b(\w+\.\w+)\b', text)
    for file_name in search_terms_origin:
        if "." in file_name:
            term_position = search_terms_origin.index(file_name)
            search_terms_old.append(file_name)
    for file_name in file_names_with_extension_old:
        if "." in file_name:
            search_terms_old.append(file_name)
    for file_name in file_names_with_extension:
        search_terms.append(file_name)
    clean_text_old = text
    clean_text = text.lower()
    search_terms_old1 = list(set(search_terms_old))
    for term in search_terms_old:
        clean_text_old = clean_text_old.replace(term, '')
    for term in search_terms:
        clean_text = clean_text.replace(term, '')
    words_old = re.findall(r'\b\w+\b', clean_text_old)
    search_terms_old.extend(words_old)
    matching_files = set()
    matching_files_old = set()
    for root, dirs, files in os.walk(folder_path):
        for file in files:
            for term in search_terms:
                if term.lower() in file.lower():
                    term_position = search_terms.index(term)
                    term_value = search_terms_origin[term_position]
                    matching_files.add(file)
                    matching_index.append(term_position)
                    break
    matching_files_old1 = []
    matching_index.sort()
    for x in matching_index:
        matching_files_old1.append(search_terms_origin[x])
    return matching_files, matching_files_old1, search_terms_old1

def convert_xlsx_to_csv(xlsx_file_path, csv_file_path):
    df = pd.read_excel(xlsx_file_path)
    df.to_csv(csv_file_path, index=False)

def save_list_CSV_id(file_list, id):
    text = ""
    for x in file_list:
        if x.endswith('.xlsx'):
            old = f"./user_file/{id}/{x}"
            new = old.replace(".xlsx", ".csv")
            convert_xlsx_to_csv(old, new)
            x = x.replace(".xlsx", ".csv")
        loader1 = CSVLoader(f"./user_file/{id}/{x}")
        docs1 = loader1.load()
        text += f"Dữ liệu file {x}:\n"
        for z in docs1:
            text += z.page_content + "\n"
    return text

def merge_files(file_set, file_list):
    """Hàm này ghép lại các tên file dựa trên điều kiện đã cho."""
    merged_files = {}
    for file_name in file_list:
        name = file_name.split('.')[0]
        for f in file_set:
            if name in f:
                merged_files[name] = f
                break
    return merged_files

def replace_keys_with_values(original_dict, replacement_dict):
    new_dict = {}
    for key, value in original_dict.items():
        if key in replacement_dict:
            new_key = replacement_dict[key]
            new_dict[new_key] = value
        else:
            new_dict[key] = value
    return new_dict

def aws1_csv_id(new_dict_csv, id):
    text = ""
    query_all = ""
    keyword = []
    for key, value in new_dict_csv.items():
        print(key, value)
        query_all += value
        keyword.append(key)
        test = save_list_CSV_id(keyword, id)
        text += test
    sources = ",".join(keyword)
    return text, query_all, sources

def chat_llama3(prompt_query):
    try:
        chat_completion = client.chat.completions.create(
            messages=[
                {
                    "role": "system",
                    "content": "Bạn là một trợ lý trung thưc, trả lời dựa trên nội dung tài liệu được cung cấp. Chỉ trả lời liên quan đến câu hỏi một cách đầy đủ chính xác, không bỏ sót thông tin."
                },
                {
                    "role": "user",
                    "content": f"{prompt_query}",
                }
            ],
            model="llama3-70b-8192",
            temperature=0.0,
            max_tokens=9000,
            stop=None,
            stream=False,
        )
        return chat_completion.choices[0].message.content
    except Exception as error:
        return False

def chat_gemini(prompt):
    generation_config = {
        "temperature": 0.0,
        "top_p": 0.0,
        "top_k": 0,
        "max_output_tokens": 8192,
    }
    safety_settings = [
        {
            "category": "HARM_CATEGORY_HARASSMENT",
            "threshold": "BLOCK_MEDIUM_AND_ABOVE"
        },
        {
            "category": "HARM_CATEGORY_HATE_SPEECH",
            "threshold": "BLOCK_MEDIUM_AND_ABOVE"
        },
        {
            "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
            "threshold": "BLOCK_MEDIUM_AND_ABOVE"
        },
        {
            "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
            "threshold": "BLOCK_MEDIUM_AND_ABOVE"
        },
    ]
    model = genai.GenerativeModel(model_name="gemini-1.5-pro-latest",
                                  generation_config=generation_config,
                                  safety_settings=safety_settings)
    convo = model.start_chat(history=[])
    convo.send_message(prompt)
    return convo.last.text

def question_answer(question):
    completion = chat_llama3(question)
    if completion:
        return completion
    else:
        answer = chat_gemini(question)
        return answer

def check_persist_directory(id, file_name):
    directory_path = f"./vector_database/{id}/{file_name}"
    return os.path.exists(directory_path)

from langchain_community.vectorstores import FAISS

def check_path_exists(path):
    return os.path.exists(path)
def aws1_all_id(new_dict, text_alls, id, thread_id):
    answer = ""
    COHERE_API_KEY1 = os.getenv("COHERE_API_KEY_1")
    os.environ["COHERE_API_KEY"] = COHERE_API_KEY1
    answer_relevant = ""
    directory = ""

    for key, value in new_dict.items():
        query = value
        keyword, keyword2, keyword3 = find_matching_files_in_docs_12_id(query, id)
        data = extract_multi_metadata_content(text_alls, keyword)
        if keyword:
            file_name = next(iter(keyword))
            text_splitter = CharacterTextSplitter(chunk_size=3200, chunk_overlap=1500)
            texts_data = text_splitter.split_text(data)

            if check_persist_directory(id, file_name):
                vectordb_query = Chroma(persist_directory=f"./vector_database/{id}/{file_name}", embedding_function=embeddings)
            else:
                vectordb_query = Chroma.from_texts(texts_data,
                                                   embedding=embeddings,
                                                   persist_directory=f"./vector_database/{id}/{file_name}")

            k_1 = len(texts_data)
            retriever = vectordb_query.as_retriever(search_kwargs={f"k": k_1})
            bm25_retriever = BM25Retriever.from_texts(texts_data)
            bm25_retriever.k = k_1
            ensemble_retriever = EnsembleRetriever(retrievers=[bm25_retriever, retriever],
                                                   weights=[0.6, 0.4])
            docs = ensemble_retriever.get_relevant_documents(f"{query}")

            path = f"./vector_database/FAISS/{id}/{file_name}"
            if check_path_exists(path):
                docsearch = FAISS.load_local(path, embeddings, allow_dangerous_deserialization=True)
            else:
                docsearch = FAISS.from_documents(docs, embeddings)
                docsearch.save_local(f"./vector_database/FAISS/{id}/{file_name}")
                docsearch = FAISS.load_local(path, embeddings, allow_dangerous_deserialization=True)

            k_2 = len(docs)
            compressor = CohereRerank(top_n=5)
            retrieve3 = docsearch.as_retriever(search_kwargs={f"k": k_2})
            compression_retriever = ContextualCompressionRetriever(
                base_compressor=compressor, base_retriever=retrieve3
            )
            compressed_docs = compression_retriever.get_relevant_documents(f"{query}")

            if compressed_docs:
                data = compressed_docs[0].page_content
                text = ''.join(map(lambda x: x.page_content, compressed_docs))
                prompt_document = f"Dựa vào nội dung sau:{text}. Hãy trả lời câu hỏi sau đây: {query}. Mà không thay đổi nội dung mà mình đã cung cấp"
                answer_for = question_answer(prompt_document)
                answer += answer_for + "\n"
                answer_relevant = data
                directory = file_name

    return answer, answer_relevant, directory

def extract_content_between_keywords(query, keywords):
    contents = {}
    num_keywords = len(keywords)
    keyword_positions = []
    for i in range(num_keywords):
        keyword = keywords[i]
        keyword_position = query.find(keyword)
        keyword_positions.append(keyword_position)
        if keyword_position == -1:
            continue
        next_keyword_position = len(query)
        for j in range(i + 1, num_keywords):
            next_keyword = keywords[j]
            next_keyword_position = query.find(next_keyword)
            if next_keyword_position != -1:
                break
        if i == 0:
            content_before = query[:keyword_position].strip()
        else:
            content_before = query[keyword_positions[i - 1] + len(keywords[i - 1]):keyword_position].strip()
        if i == num_keywords - 1:
            content_after = query[keyword_position + len(keyword):].strip()
        else:
            content_after = query[keyword_position + len(keyword):next_keyword_position].strip()
        content = f"{content_before} {keyword} {content_after}"
        contents[keyword] = content
    return contents

def generate_random_questions(filtered_ques_list):
    if len(filtered_ques_list) >= 2:
        random_questions = random.sample(filtered_ques_list, 2)
    else:
        random_questions = filtered_ques_list
    return random_questions

def generate_question_main(loader, name_file):
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=4500, chunk_overlap=2500)
    texts = text_splitter.split_documents(loader)
    question_gen = f"nội dung {name_file} : \n"
    question_gen += texts[0].page_content
    splitter_ques_gen = RecursiveCharacterTextSplitter(
        chunk_size=4500,
        chunk_overlap=2200
    )
    chunks_ques_gen = splitter_ques_gen.split_text(question_gen)
    document_ques_gen = [Document(page_content=t) for t in chunks_ques_gen]
    llm_ques_gen_pipeline = llm
    prompt_template_vn = """
    Bạn là một chuyên gia tạo câu hỏi dựa trên tài liệu và tài liệu hướng dẫn.
    Bạn làm điều này bằng cách đặt các câu hỏi về đoạn văn bản dưới đây:

    ------------
    {text}
    ------------

    Hãy tạo ra các câu hỏi từ đoạn văn bản này.Nếu đoạn văn là tiếng Việt hãy tạo câu hỏi tiếng Việt. Nếu đoạn văn là tiếng Anh hãy tạo câu hỏi tiếng Anh.
    Hãy chắc chắn không bỏ sót bất kỳ thông tin quan trọng nào. Và chỉ tạo với đoạn tài liệu đó tối đa 5 câu hỏi liên quan tới tài liệu cung cấp nhất.Nếu trong đoạn tài liệu có các tên liên quan đến file như demo1.pdf( nhiều file khác) thì phải kèm nó vào nội dung câu hỏi bạn tạo ra.

    CÁC CÂU HỎI:
    """

    PROMPT_QUESTIONS_VN = PromptTemplate(template=prompt_template_vn, input_variables=["text"])
    refine_template_vn = ("""
    Bạn là một chuyên gia tạo câu hỏi thực hành dựa trên tài liệu và tài liệu hướng dẫn.
    Mục tiêu của bạn là giúp người học chuẩn bị cho một kỳ thi.
    Chúng tôi đã nhận được một số câu hỏi thực hành ở mức độ nào đó: {existing_answer}.
    Chúng tôi có thể tinh chỉnh các câu hỏi hiện có hoặc thêm câu hỏi mới
    (chỉ khi cần thiết) với một số ngữ cảnh bổ sung dưới đây.
    ------------
    {text}
    ------------

    Dựa trên ngữ cảnh mới, hãy tinh chỉnh các câu hỏi bằng tiếng Việt nếu đoạn văn đó cung cấp tiếng Việt. Nếu không hãy tinh chỉnh câu hỏi bằng tiếng Anh nếu đoạn đó cung cấp tiếng Anh.
    Nếu ngữ cảnh không hữu ích, vui lòng cung cấp các câu hỏi gốc. Và chỉ tạo với đoạn tài liệu đó tối đa 5 câu hỏi liên quan tới tài liệu cung cấp nhất. Nếu trong đoạn tài liệu có các tên file thì phải kèm nó vào câu hỏi.
    CÁC CÂU HỎI:
    """
                          )

    REFINE_PROMPT_QUESTIONS = PromptTemplate(
        input_variables=["existing_answer", "text"],
        template=refine_template_vn,
    )
    ques_gen_chain = load_summarize_chain(llm=llm_ques_gen_pipeline,
                                          chain_type="refine",
                                          verbose=True,
                                          question_prompt=PROMPT_QUESTIONS_VN,
                                          refine_prompt=REFINE_PROMPT_QUESTIONS)
    ques = ques_gen_chain.run(document_ques_gen)
    ques_list = ques.split("\n")
    filtered_ques_list = ["{}: {}".format(name_file, re.sub(r'^\d+\.\s*', '', element)) for element in ques_list if
                          element.endswith('?') or element.endswith('.')]
    return generate_random_questions(filtered_ques_list)

def load_file(loader):
    return loader.load()

def extract_data2(id):
    documents = []
    directory_path = f"./user_file/{id}"
    if not os.path.exists(directory_path) or not any(
            os.path.isfile(os.path.join(directory_path, f)) for f in os.listdir(directory_path)):
        return False
    tasks = []
    with ThreadPoolExecutor() as executor:
        for file in os.listdir(directory_path):
            if file.endswith(".pdf"):
                pdf_path = os.path.join(directory_path, file)
                loader = UnstructuredPDFLoader(pdf_path)
                tasks.append(executor.submit(load_file, loader))
            elif file.endswith('.docx') or file.endswith('.doc'):
                doc_path = os.path.join(directory_path, file)
                loader = Docx2txtLoader(doc_path)
                tasks.append(executor.submit(load_file, loader))
            elif file.endswith('.txt'):
                txt_path = os.path.join(directory_path, file)
                loader = TextLoader(txt_path, encoding="utf8")
                tasks.append(executor.submit(load_file, loader))
            elif file.endswith('.pptx'):
                ppt_path = os.path.join(directory_path, file)
                loader = UnstructuredPowerPointLoader(ppt_path)
                tasks.append(executor.submit(load_file, loader))
            elif file.endswith('.csv'):
                csv_path = os.path.join(directory_path, file)
                loader = UnstructuredCSVLoader(csv_path)
                tasks.append(executor.submit(load_file, loader))
            elif file.endswith('.xlsx'):
                excel_path = os.path.join(directory_path, file)
                loader = UnstructuredExcelLoader(excel_path)
                tasks.append(executor.submit(load_file, loader))
            elif file.endswith('.json'):
                json_path = os.path.join(directory_path, file)
                loader = TextLoader(json_path)
                tasks.append(executor.submit(load_file, loader))
            elif file.endswith('.md'):
                md_path = os.path.join(directory_path, file)
                loader = UnstructuredMarkdownLoader(md_path)
                tasks.append(executor.submit(load_file, loader))
        for future in as_completed(tasks):
            result = future.result()
            documents.extend(result)
    text_splitter = CharacterTextSplitter(chunk_size=4500, chunk_overlap=2500)
    texts = text_splitter.split_documents(documents)
    Chroma.from_documents(documents=texts,
                          embedding=embeddings,
                          persist_directory=f"./vector_database/{id}")
    return texts

def generate_question(id):
    directory_path = f"./user_file/{id}"
    if not os.path.exists(directory_path) or not any(
            os.path.isfile(os.path.join(directory_path, f)) for f in os.listdir(directory_path)):
        return False
    all_questions = []
    tasks = []
    with ThreadPoolExecutor() as executor:
        for file in os.listdir(directory_path):
            if file.endswith(".pdf"):
                pdf_path = os.path.join(directory_path, file)
                loader = UnstructuredPDFLoader(pdf_path).load()
                tasks.append(executor.submit(generate_question_main, loader, file))
            elif file.endswith('.docx') or file.endswith('.doc'):
                doc_path = os.path.join(directory_path, file)
                loader = Docx2txtLoader(doc_path).load()
                tasks.append(executor.submit(generate_question_main, loader, file))
            elif file.endswith('.txt'):
                txt_path = os.path.join(directory_path, file)
                loader = TextLoader(txt_path, encoding="utf8").load()
                tasks.append(executor.submit(generate_question_main, loader, file))
            elif file.endswith('.pptx'):
                ppt_path = os.path.join(directory_path, file)
                loader = UnstructuredPowerPointLoader(ppt_path).load()
                tasks.append(executor.submit(generate_question_main, loader, file))
            elif file.endswith('.json'):
                json_path = os.path.join(directory_path, file)
                loader = TextLoader(json_path, encoding="utf8").load()
                tasks.append(executor.submit(generate_question_main, loader, file))
            elif file.endswith('.md'):
                md_path = os.path.join(directory_path, file)
                loader = UnstructuredMarkdownLoader(md_path).load()
                tasks.append(executor.submit(generate_question_main, loader, file))
        for future in as_completed(tasks):
            result = future.result()
            all_questions.extend(result)
    return all_questions

class Search(BaseModel):
    queries: List[str] = Field(
        ...,
        description="Truy vấn riêng biệt để tìm kiếm, giữ nguyên ý chính câu hỏi riêng biệt",
    )

def query_analyzer(query):
    output_parser = PydanticToolsParser(tools=[Search])
    system = """Bạn có khả năng đưa ra các truy vấn tìm kiếm chính xác để lấy thông tin giúp trả lời các yêu cầu của người dùng. Các truy vấn của bạn phải chính xác, không được bỏ ngắn rút gọn.
     Nếu bạn cần tra cứu hai hoặc nhiều thông tin riêng biệt, bạn có thể làm điều đó!. Trả lời câu hỏi bằng tiếng Việt(Vietnamese), không được dùng ngôn ngữ khác"""
    prompt = ChatPromptTemplate.from_messages(
        [
            ("system", system),
            ("human", "{question}"),
        ]
    )
    llm = ChatOpenAI(model="gpt-3.5-turbo-0125", temperature=0.0)
    structured_llm = llm.with_structured_output(Search)
    query_analyzer = {"question": RunnablePassthrough()} | prompt | structured_llm
    text = query_analyzer.invoke(query)
    return text

def handle_query(question, text_all, compression_retriever, id, thread_id):
        query = question
        x = query
        keyword, key_words_old, key3 = find_matching_files_in_docs_12_id(query, id)
        if keyword == set() or key_words_old == list():
            return "Not found file"
        file_list = keyword

        if file_list:
            list_keywords2 = list(key_words_old)
            contents1 = extract_content_between_keywords(query, list_keywords2)
            merged_result = merge_files(keyword, list_keywords2)
            original_dict = contents1
            replacement_dict = merged_result
            new_dict = replace_keys_with_values(original_dict, replacement_dict)
            files_to_remove = [filename for filename in new_dict.keys() if
                               filename.endswith('.xlsx') or filename.endswith('.csv')]
            removed_files = {}
            for filename in files_to_remove:
                removed_files[filename] = new_dict[filename]
            for filename in files_to_remove:
                new_dict.pop(filename)
            test_csv = ""
            text_csv, query_csv, source = aws1_csv_id(removed_files, id)
            prompt_csv = ""
            answer_csv = ""
            if test_csv:
                prompt_csv = f"Dựa vào nội dung sau: {text_csv}. Hãy trả lời câu hỏi sau đây: {query_csv}.Bằng tiếng Việt"
                answer_csv = question_answer(prompt_csv)
            answer_document, data_relevant, source = aws1_all_id(new_dict, text_all, id, thread_id)
            answer_all1 = answer_document + answer_csv
            return answer_all1, data_relevant, source
        else:
            compressed_docs = compression_retriever.get_relevant_documents(f"{query}")
            relevance_score_float = float(compressed_docs[0].metadata['relevance_score'])
            if relevance_score_float <= 0.25:
                documents1 = []
                for file in os.listdir(f"./user_file/{id}"):
                    if file.endswith('.csv'):
                        csv_path = f"./user_file/{id}/" + file
                        loader = UnstructuredCSVLoader(csv_path)
                        documents1.extend(loader.load())
                    elif file.endswith('.xlsx'):
                        excel_path = f"./user_file/{id}/" + file
                        loader = UnstructuredExcelLoader(excel_path)
                        documents1.extend(loader.load())
                text_splitter_csv = CharacterTextSplitter.from_tiktoken_encoder(chunk_size=2200, chunk_overlap=1500)
                texts_csv = text_splitter_csv.split_documents(documents1)
                vectordb_csv = Chroma.from_documents(documents=texts_csv,
                                                     embedding=embeddings, persist_directory=f'./vector_database/csv/{thread_id}')
                k = len(texts_csv)
                retriever_csv = vectordb_csv.as_retriever(search_kwargs={"k": k})
                llm = Cohere(temperature=0)
                compressor_csv = CohereRerank(top_n=3, model="rerank-english-v2.0")
                compression_retriever_csv = ContextualCompressionRetriever(
                    base_compressor=compressor_csv, base_retriever=retriever_csv
                )
                compressed_docs_csv = compression_retriever_csv.get_relevant_documents(f"{query}")
                file_path = compressed_docs_csv[0].metadata['source']
                print(file_path)
                if file_path.endswith('.xlsx'):
                    new = file_path.replace(".xlsx", ".csv")
                    convert_xlsx_to_csv(file_path, new)
                    loader1 = CSVLoader(new)
                else:
                    loader1 = CSVLoader(file_path)
                docs1 = loader1.load()
                text = " "
                for z in docs1:
                    text += z.page_content + "\n"
                prompt_csv = f"Dựa vào nội dung sau: {text}. Hãy trả lời câu hỏi sau đây: {query}. Bằng tiếng Việt"
                answer_csv = question_answer(prompt_csv)
                return answer_csv
            else:
                file_path = compressed_docs[0].metadata['source']
                if file_path.endswith(".pdf"):
                    loader = UnstructuredPDFLoader(file_path)
                elif file_path.endswith('.docx') or file_path.endswith('doc'):
                    loader = Docx2txtLoader(file_path)
                elif file_path.endswith('.txt'):
                    loader = TextLoader(file_path, encoding="utf8")
                elif file_path.endswith('.pptx'):
                    loader = UnstructuredPowerPointLoader(file_path)
                elif file_path.endswith('.xml'):
                    loader = UnstructuredXMLLoader(file_path)
                elif file_path.endswith('.html'):
                    loader = UnstructuredHTMLLoader(file_path)
                elif file_path.endswith('.json'):
                    loader = TextLoader(file_path)
                elif file_path.endswith('.md'):
                    loader = UnstructuredMarkdownLoader(file_path)
                elif file_path.endswith('.xlsx'):
                    file_path_new = file_path.replace(".xlsx", ".csv")
                    convert_xlsx_to_csv(file_path, file_path_new)
                    loader = CSVLoader(file_path_new)
                elif file_path.endswith('.csv'):
                    loader = CSVLoader(file_path)
                text_splitter = CharacterTextSplitter(chunk_size=3200, chunk_overlap=1500)
                texts = text_splitter.split_documents(loader.load())
                k_1 = len(texts)
                file_name = os.path.basename(file_path)
                if check_persist_directory(id, file_name):
                    vectordb_file = Chroma(persist_directory=f"./vector_database/{id}/{file_name}",
                                           embedding_function=embeddings)
                else:
                    vectordb_file = Chroma.from_documents(texts,
                                                          embedding=embeddings,
                                                          persist_directory=f"./vector_database/{id}/{file_name}")
                retriever_file = vectordb_file.as_retriever(search_kwargs={f"k": k_1})
                compressor_file = CohereRerank(top_n=5, model="rerank-english-v2.0")
                compression_retriever_file = ContextualCompressionRetriever(
                    base_compressor=compressor_file, base_retriever=retriever_file
                )
                compressed_docs_file = compression_retriever_file.get_relevant_documents(f"{x}")
                query = question
                text = ''.join(map(lambda x: x.page_content, compressed_docs_file))
                prompt = f"Dựa vào nội dung sau:{text}. Hãy trả lời câu hỏi sau đây: {query}. Mà không thay đổi, chỉnh sửa nội dung mà mình đã cung cấp"
                answer = question_answer(prompt)
                list_relevant = compressed_docs_file[0].page_content
                source = file_name
                return answer, list_relevant, source

def handle_query_upgrade_keyword_old(query_all, text_all, id):
    COHERE_API_KEY_2 = os.environ["COHERE_API_KEY_2"]
    os.environ["COHERE_API_KEY"] = COHERE_API_KEY_2
    test = query_analyzer(query_all)
    test_string = str(test)
    matches = re.findall(r"'([^']*)'", test_string)
    vectordb = Chroma(persist_directory=f"./vector_database/{id}", embedding_function=embeddings)
    k = len(text_all)
    retriever = vectordb.as_retriever(search_kwargs={"k": k})
    compressor = CohereRerank(top_n=5, model="rerank-english-v2.0")
    compression_retriever = ContextualCompressionRetriever(base_compressor=compressor, base_retriever= retriever)
    with ThreadPoolExecutor() as executor:
        futures = {executor.submit(handle_query, query, text_all, compression_retriever, id, i): query for i, query in
                   enumerate(matches)}
        results = []
        data_relevant = []
        sources = []
        for future in as_completed(futures):
            try:
                result, list_data, list_source = future.result()
                results.append(result)
                data_relevant.append(list_data)
                sources.append(list_source)
            except Exception as e:
                print(f'An error occurred: {e}')
    answer_all = ''.join(results)
    prompt1 = f"Dựa vào nội dung sau:{answer_all}. Hãy trả lời câu hỏi sau đây: {query_all}. Mà không thay đổi, chỉnh sửa nội dung mà mình đã cung cấp"
    answer1 = question_answer(prompt1)
    return answer1, data_relevant, sources