Upload 5 files

app.py

@@ -0,0 +1,1022 @@
+# Chat with Documents 2 by cawacci
+# 2023.9.10 キカガク長期コース（2023年4月期）の成果物アプリとして制作
+
+# --------------------------------------
+# Libraries
+# --------------------------------------
+import os
+import time
+import gc # メモリ解放
+import re # 正規表現で文章をクリーンアップ
+
+# HuggingFace
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+# OpenAI
+import openai
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.chat_models import ChatOpenAI
+
+# LangChain
+from langchain.llms import HuggingFacePipeline
+from transformers import pipeline
+
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.chains import LLMChain, VectorDBQA
+from langchain.vectorstores import Chroma
+
+from langchain import PromptTemplate, ConversationChain
+from langchain.chains.question_answering import load_qa_chain # QA Chat
+from langchain.document_loaders import SeleniumURLLoader # URL取得
+from langchain.docstore.document import Document # テキストをドキュメント化
+from langchain.memory import ConversationSummaryBufferMemory # チャット履歴
+
+from typing import Any
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+from langchain.tools import DuckDuckGoSearchRun
+
+# Gradio
+import gradio as gr
+from pypdf import PdfReader
+import requests # DeepL API request
+
+# test
+import langchain # (debug=Trueにするため)
+
+# --------------------------------------
+#  ユーザ別セッションの変数値を記録するクラス
+# 　（参考）https://blog.shikoan.com/gradio-state/
+# --------------------------------------
+class SessionState:
+	def __init__(self):
+		# Hugging Face
+		self.tokenizer						= None
+		self.pipe									= None
+		self.model								= None
+
+		# LangChain
+		self.llm									= None
+		self.embeddings						= None
+		self.current_model				= ""
+		self.current_embedding		= ""
+		self.db										= None		# Vector DB
+		self.memory								= None		# Langchain Chat Memory
+		self.conversation_chain		= None		# ConversationChain
+		self.query_generator			= None		# Query Refiner with Chat history
+		self.qa_chain							= None		# load_qa_chain
+		self.embedded_urls				= []
+		self.similarity_search_k	= None		# No. of similarity search documents to find.
+		self.summarization_mode		=	None		# Stuff / Map Reduce / Refine
+
+		# Apps
+		self.dialogue							= []			# Recent Chat History for display
+
+	# --------------------------------------
+	# Empty Cache
+	# --------------------------------------
+	def cache_clear(self):
+		if torch.cuda.is_available():
+			torch.cuda.empty_cache()	# GPU Memory Clear
+
+		gc.collect()								# CPU Memory Clear
+
+	# --------------------------------------
+	# Clear Models (llm: llm model, embd: embeddings, db: vectordb)
+	# --------------------------------------
+	def clear_memory(self, llm=False, embd=False, db=False):
+		# DB
+		if db and self.db:
+			self.db.delete_collection()
+			self.db									= None
+			self.embedded_urls			= []
+
+		# Embeddings model
+		if llm or embd:
+			self.embeddings 				= None
+			self.current_embedding	= ""
+			self.qa_chain 					= None
+
+		# LLM model
+		if llm:
+			self.llm								= None
+			self.pipe								= None
+			self.model							= None
+			self.current_model			= ""
+			self.tokenizer					= None
+			self.memory 						= None
+			self.chat_history				= []			# ←必要性を要検証
+
+		self.cache_clear()
+
+# --------------------------------------
+# 自作TextSplitter（テキストをLLMのトークン数内に分割）
+# （参考）https://www.sato-susumu.com/entry/2023/04/30/131338
+# 　→　「！」、「？」、「）」、「.」、「!」、「?」、「,」などを追加
+# --------------------------------------
+class JPTextSplitter(RecursiveCharacterTextSplitter):
+    def __init__(self, **kwargs: Any):
+        separators = ["\n\n", "\n", "。", "！", "？", "）","、", ".", "!", "?", ",", " ", ""]
+        super().__init__(separators=separators, **kwargs)
+
+# チャンクの分割
+chunk_size    = 512
+chunk_overlap = 35
+
+text_splitter = JPTextSplitter(
+    chunk_size    = chunk_size,  # チャンクの最大文字数
+    chunk_overlap = chunk_overlap,  # オーバーラップの最大文字数
+)
+
+# --------------------------------------
+# DeepL でメモリを翻訳しトークン数を削減(OpenAIモデル利用時)
+# --------------------------------------
+DEEPL_API_ENDPOINT = "https://api-free.deepl.com/v2/translate"
+DEEPL_API_KEY = os.getenv("DEEPL_API_KEY")
+
+def deepl_memory(ss: SessionState) -> (SessionState):
+  if ss.current_model == "gpt-3.5-turbo":
+    # メモリから会話履歴を取得
+    user_message = ss.memory.chat_memory.messages[-2].content
+    ai_message = ss.memory.chat_memory.messages[-1].content
+    text = [user_message, ai_message]
+
+    # DeepL設定
+    params = {
+        "auth_key": DEEPL_API_KEY,
+        "text": text,
+        "target_lang": "EN",
+        "source_lang": "JA",
+        "tag_handling": "xml",
+        "igonere_tags": "x",
+    }
+    request = requests.post(DEEPL_API_ENDPOINT, data=params)
+    request.raise_for_status()  # 応答のステータスコードがエラーの場合は例外を発生させます。
+    response = request.json()
+
+    # JSONから翻訳文を取得
+    user_message = response["translations"][0]["text"]
+    ai_message = response["translations"][1]["text"]
+
+    # memoryの最後の会話を削除し、翻訳文を追加
+    ss.memory.chat_memory.messages = ss.memory.chat_memory.messages[:-2]
+    ss.memory.chat_memory.add_user_message(user_message)
+    ss.memory.chat_memory.add_ai_message(ai_message)
+
+  return ss
+
+# --------------------------------------
+# DuckDuckGo Web検索結果を入力プロンプトに追加
+# --------------------------------------
+# DEEPL_API_ENDPOINT = "https://api-free.deepl.com/v2/translate"
+# DEEPL_API_KEY = os.getenv("DEEPL_API_KEY")
+
+def web_search(query, current_model) -> str:
+  search =  DuckDuckGoSearchRun()
+  web_result = search(query)
+
+  if current_model == "gpt-3.5-turbo":
+    text = [query, web_result]
+    params = {
+        "auth_key": DEEPL_API_KEY,
+        "text": text,
+        "target_lang": "EN",
+        "source_lang": "JA",
+        "tag_handling": "xml",
+        "igonere_tags": "x",
+    }
+    request = requests.post(DEEPL_API_ENDPOINT, data=params)
+    response = request.json()
+
+    query = response["translations"][0]["text"]
+    web_result =  response["translations"][1]["text"]
+
+  web_query = query + "\nUse the following information as a reference to answer the question above in the Japanese.\n===\nReference: " + web_result + "\n==="
+
+  return web_query
+
+# --------------------------------------
+# LangChain カスタムプロンプト各種
+#   llama tokenizer
+#   https://belladoreai.github.io/llama-tokenizer-js/example-demo/build/
+
+#   OpenAI tokenizer
+#   https://platform.openai.com/tokenizer
+# --------------------------------------
+
+# --------------------------------------
+# Conversation Chain Template
+# --------------------------------------
+
+# Tokens: OpenAI 104/ Llama 105 <- In Japanese: Tokens: OpenAI 191/ Llama 162
+sys_chat_message = """
+The following is a conversation between an AI concierge and a customer.
+The AI understands what the customer wants to know from the conversation history and the latest question,
+and gives many specific details in Japanese. If the AI does not know the answer to a question, it does not
+make up an answer and says "誠に申し訳ございませんが、その点についてはわかりかねます".
+""".replace("\n", "")
+
+chat_common_format = """
+===
+Question: {query}
+
+Conversation History:
+{chat_history}
+
+日本語の回答: """
+
+chat_template_std = f"{sys_chat_message}{chat_common_format}"
+chat_template_llama2 = f"<s>[INST] <<SYS>>{sys_chat_message}<</SYS>>{chat_common_format}[/INST]"
+
+# --------------------------------------
+# QA Chain Template (Stuff)
+# --------------------------------------
+# Tokens: OpenAI 113/ Llama 111 <- In Japanese: Tokens: OpenAI 256/ Llama 225
+sys_qa_message = """
+You are an AI concierge who carefully answers questions from customers based on references.
+You understand what the customer wants to know from the Conversation History and Question,
+and give a specific answer in Japanese using sentences extracted from the following references.
+If you do not know the answer, do not make up an answer and reply,
+"誠に申し訳ございませんが、その点についてはわかりかねます".
+""".replace("\n", "")
+
+qa_common_format = """
+===
+Question: {query}
+References: {context}
+Conversation History:
+{chat_history}
+
+日本語の回答: """
+
+qa_template_std = f"{sys_qa_message}{qa_common_format}"
+qa_template_llama2 = f"<s>[INST] <<SYS>>{sys_qa_message}<</SYS>>{qa_common_format}[/INST]"
+
+# --------------------------------------
+# QA Chain Template (Map Reduce)
+# --------------------------------------
+# 1. 会話履歴と最新の質問から、質問文を生成するchain のプロンプト
+query_generator_message = """
+Referring to the "Conversation History", reformat the user's "Additional Question"
+to a specific question in Japanese by filling in the missing subject, verb, objects,
+complements, and other necessary information to get a better search result.
+""".replace("\n", "")
+
+query_generator_common_format = """
+===
+[Conversation History]
+{chat_history}
+
+[Additional Question] {query}
+明確な質問文: """
+
+query_generator_template_std = f"{query_generator_message}{query_generator_common_format}"
+query_generator_template_llama2 = f"<s>[INST] <<SYS>>{query_generator_message}<</SYS>>{query_generator_common_format}[/INST]"
+
+
+# 2. 生成された質問文を用いて、参考文献を要約するchain のプロンプト
+question_prompt_message = """
+From the following references, extract key information relevant to the question
+and summarize it in a natural English sentence with clear subject, verb, object,
+and complement.
+""".replace("\n", "")
+
+question_prompt_common_format = """
+===
+[references] {context}
+[Question] {query}
+[Summary] """
+
+question_prompt_template_std = f"{question_prompt_message}{question_prompt_common_format}"
+question_prompt_template_llama2 = f"<s>[INST] <<SYS>>{question_prompt_message}<</SYS>>{question_prompt_common_format}[/INST]"
+
+
+# 3. 生成された質問文とベクターデータベースの要約をもとに、回答を行うchain のプロンプト
+combine_prompt_message = """
+You are an AI concierge who carefully answers questions from customers based on references.
+Provide a specific answer in Japanese using sentences extracted from the following references.
+If you do not know the answer, do not make up an answer and reply,
+"誠に申し訳ございませんが、その点についてはわかりかねます".
+""".replace("\n", "")
+
+combine_prompt_common_format = """
+===
+Question:
+{query}
+===
+Reference: {summaries}
+===
+日本語の回答: """
+
+combine_prompt_template_std = f"{combine_prompt_message}{combine_prompt_common_format}"
+combine_prompt_template_llama2 = f"<s>[INST] <<SYS>>{combine_prompt_message}<</SYS>>{combine_prompt_common_format}[/INST]"
+
+
+# --------------------------------------
+# ConversationSummaryBufferMemoryの要約プロンプト
+# ソース　→　https://github.com/langchain-ai/langchain/blob/894c272a562471aadc1eb48e4a2992923533dea0/langchain/memory/prompt.py#L26-L49
+# --------------------------------------
+# Tokens: OpenAI 212/ Llama 214 <- In Japanese: Tokens: OpenAI 397/ Llama 297
+conversation_summary_template = """
+Using the example as a guide, compose a summary in English that gives an overview of the conversation by summarizing the "current summary" and the "new conversation".
+===
+Example
+[Current Summary] Customer asks AI what it thinks about Artificial Intelligence, AI says Artificial Intelligence is a good tool.
+
+[New Conversation]
+Human: なぜ人工知能が良いツールだと思いますか？
+AI: 人工知能は「人間の可能性を最大限に引き出すことを助ける」からです。
+
+[New Summary] Customer asks what you think about Artificial Intelligence, and AI responds that it is a good force that helps humans reach their full potential.
+===
+[Current Summary] {summary}
+
+[New Conversation]
+{new_lines}
+
+[New Summary]
+""".strip()
+
+# モデル読み込み
+def load_models(
+  ss: SessionState,
+  model_id: str,
+  embedding_id: str,
+  openai_api_key: str,
+  load_in_8bit: bool,
+  verbose: bool,
+  temperature: float,
+  similarity_search_k: int,
+  summarization_mode: str,
+  min_length: int,
+  max_new_tokens: int,
+  top_k: int,
+  top_p: float,
+  repetition_penalty: float,
+  num_return_sequences: int,
+) -> (SessionState, str):
+
+  # --------------------------------------
+  # 変数の保存
+  # --------------------------------------
+  ss.similarity_search_k  = similarity_search_k
+  ss.summarization_mode   = summarization_mode
+
+  # --------------------------------------
+  # OpenAI API KEYの確認
+  # --------------------------------------
+  if (model_id == "gpt-3.5-turbo" or embedding_id == "text-embedding-ada-002"):
+    # 前処理
+    if not os.environ["OPENAI_API_KEY"]:
+      status_message =  "❌ OpenAI API KEY を設定してください"
+      return ss, status_message
+
+  # --------------------------------------
+  # LLMの設定
+  # --------------------------------------
+  # OpenAI Model
+  if model_id == "gpt-3.5-turbo":
+    ss.clear_memory(llm=True, db=True)
+    ss.llm = ChatOpenAI(
+      model_name    = model_id,
+      temperature   = temperature,
+      verbose       = verbose,
+      max_tokens    = max_new_tokens,
+    )
+
+  # Hugging Face GPT Model
+  else:
+    ss.clear_memory(llm=True, db=True)
+
+    if model_id == "rinna/bilingual-gpt-neox-4b-instruction-sft":
+      ss.tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=False)
+    else:
+      ss.tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+    ss.model = AutoModelForCausalLM.from_pretrained(
+      model_id,
+      load_in_8bit    = load_in_8bit,
+      torch_dtype     = torch.float16,
+      device_map      = "auto",
+    )
+
+    ss.pipe = pipeline(
+      "text-generation",
+      model                 = ss.model,
+      tokenizer             = ss.tokenizer,
+      min_length            = min_length,
+      max_new_tokens        = max_new_tokens,
+      do_sample             = True,
+      top_k                 = top_k,
+      top_p                 = top_p,
+      repetition_penalty    = repetition_penalty,
+      num_return_sequences  = num_return_sequences,
+      temperature           = temperature,
+    )
+    ss.llm = HuggingFacePipeline(pipeline=ss.pipe)
+
+  # --------------------------------------
+  # 埋め込みモデルの設定
+  # --------------------------------------
+  if ss.current_embedding == embedding_id:
+    pass
+
+  else:
+    # Reset embeddings and vectordb
+    ss.clear_memory(embd=True, db=True)
+
+    if embedding_id == "None":
+      pass
+
+    # OpenAI
+    elif embedding_id == "text-embedding-ada-002":
+      ss.embeddings = OpenAIEmbeddings()
+
+    # Hugging Face
+    else:
+      ss.embeddings = HuggingFaceEmbeddings(model_name=embedding_id)
+
+  # --------------------------------------
+  # チェーンの設定
+  #---------------------------------------
+  ss = set_chains(ss, summarization_mode)
+
+  # --------------------------------------
+  # 現在のモデル名を SessionStateオブジェクトに保存
+  #---------------------------------------
+  ss.current_model = model_id
+  ss.current_embedding = embedding_id
+
+  # Status Message
+  status_message = "✅ LLM: " + ss.current_model + ", embeddings: " + ss.current_embedding
+
+  return ss, status_message
+
+# --------------------------------------
+# Conversation/QA Chain 呼び出し統合
+# --------------------------------------
+def set_chains(ss: SessionState, summarization_mode) -> SessionState:
+
+  # モデルに合わせて chat_template を設定
+  human_prefix              = "Human: "
+  ai_prefix                 = "AI: "
+  chat_template             = chat_template_std
+  qa_template               = qa_template_std
+  query_generator_template  = query_generator_template_std
+  question_template         = question_prompt_template_std
+  combine_template          = combine_prompt_template_std
+
+  if ss.current_model == "rinna/bilingual-gpt-neox-4b-instruction-sft":
+    # Rinnaモデル向けの設定（改行コード修正、メモリ用prefix (公式ページ参照)
+    chat_template             = chat_template.replace("\n", "<NL>")
+    qa_template               = qa_template.replace("\n", "<NL>")
+    query_generator_template  = query_generator_template_std.replace("\n", "<NL>")
+    question_template         = question_prompt_template_std.replace("\n", "<NL>")
+    combine_template          = combine_prompt_template_std.replace("\n", "<NL>")
+    human_prefix              = "ユーザー: "
+    ai_prefix                 = "システム: "
+
+  elif ss.current_model.startswith("elyza/ELYZA-japanese-Llama-2-7b"):
+    # ELYZAモデル向けのテンプレート設定
+    chat_template             = chat_template_llama2
+    qa_template               = qa_template_llama2
+    query_generator_template  = query_generator_template_llama2
+    question_template         = question_prompt_template_llama2
+    combine_template          = combine_prompt_template_llama2
+
+  # --------------------------------------
+  # メモリの設定
+  # --------------------------------------
+  if ss.memory is None:
+    conversation_summary_prompt = PromptTemplate(input_variables=['summary', 'new_lines'], template=conversation_summary_template)
+    ss.memory = ConversationSummaryBufferMemory(
+        llm             = ss.llm,
+        memory_key      = "chat_history",
+        input_key       = "query",
+        output_key      = "output_text",
+        return_messages = False,
+        human_prefix    = human_prefix,
+        ai_prefix       = ai_prefix,
+        max_token_limit = 1024,
+        prompt          = conversation_summary_prompt,
+    )
+
+  # --------------------------------------
+  # Conversation/QAチェーンの設定
+  # --------------------------------------
+  if ss.conversation_chain is None:
+    chat_prompt = PromptTemplate(input_variables=['query', 'chat_history'], template=chat_template)
+    ss.conversation_chain = ConversationChain(
+      llm         = ss.llm,
+      prompt      = chat_prompt,
+      memory      = ss.memory,
+      input_key   = "query",
+      output_key  = "output_text",
+      verbose     = True,
+    )
+
+  if ss.qa_chain is None:
+    if summarization_mode == "stuff":
+      qa_prompt               = PromptTemplate(input_variables=['context', 'query', 'chat_history'], template=qa_template)
+      ss.qa_chain             = load_qa_chain(ss.llm, chain_type="stuff", memory=ss.memory, prompt=qa_prompt)
+
+    elif summarization_mode == "map_reduce":
+      query_generator_prompt  = PromptTemplate(template=query_generator_template, input_variables = ["chat_history", "query"])
+      ss.query_generator      = LLMChain(llm=ss.llm, prompt=query_generator_prompt)
+
+      question_prompt         = PromptTemplate(template=question_template, input_variables=["context", "query"])
+      combine_prompt          = PromptTemplate(template=combine_template, input_variables=["summaries", "query"])
+      ss.qa_chain             = load_qa_chain(ss.llm, chain_type="map_reduce", return_map_steps=True, memory=ss.memory, question_prompt=question_prompt, combine_prompt=combine_prompt)
+
+  return ss
+
+def initialize_db(ss: SessionState) -> SessionState:
+
+  # client = chromadb.PersistentClient(path="./db")
+  ss.db = Chroma(
+      collection_name = "user_reference",
+      embedding_function = ss.embeddings,
+      # client = client
+  )
+
+  return ss
+
+def embedding_process(ss: SessionState, ref_documents: Document) -> SessionState:
+
+  # --------------------------------------
+  # 文章構成と不要な文字列の削除
+  # --------------------------------------
+  for i in range(len(ref_documents)):
+    content = ref_documents[i].page_content.strip()
+
+    # --------------------------------------
+    # PDFの場合は読み取りエラー対策で文書修正を強めに実施
+    # --------------------------------------
+    if ".pdf" in ref_documents[i].metadata['source']:
+      pdf_replacement_sets = [
+        ('\n ', '**PLACEHOLDER+SPACE**'),
+        ('\n\u3000', '**PLACEHOLDER+SPACE**'),
+        ('.\n', '。**PLACEHOLDER**'),
+        (',\n', '。**PLACEHOLDER**'),
+        ('?\n', '。**PLACEHOLDER**'),
+        ('!\n', '。**PLACEHOLDER**'),
+        ('!\n', '。**PLACEHOLDER**'),
+        ('。\n', '。**PLACEHOLDER**'),
+        ('！\n', '！**PLACEHOLDER**'),
+        (')\n', '！**PLACEHOLDER**'),
+        (']\n', '！**PLACEHOLDER**'),
+        ('？\n', '？**PLACEHOLDER**'),
+        ('）\n', '？**PLACEHOLDER**'),
+        ('】\n', '？**PLACEHOLDER**'),
+      ]
+      for original, replacement in pdf_replacement_sets:
+        content = content.replace(original, replacement)
+      content = content.replace(" ", "")
+    # --------------------------------------
+
+    # 不要文字列・空白の削除
+    remove_texts = ["\n", "\r", "  "]
+    for remove_text in remove_texts:
+      content = content.replace(remove_text, "")
+
+    # タブや連続空白をシングルスペースに変換
+    replace_texts = ["\t", "\u3000"]
+    for replace_text in replace_texts:
+      content = content.replace(replace_text, " ")
+
+    # PDFの正当な改行をもとに戻す。
+    if ".pdf" in ref_documents[i].metadata['source']:
+      content = content.replace('**PLACEHOLDER**', '\n').replace('**PLACEHOLDER+SPACE**', '\n ')
+
+    ref_documents[i].page_content = content
+
+  # --------------------------------------
+  # チャンクに分割
+  texts = text_splitter.split_documents(ref_documents)
+
+  # --------------------------------------
+  # multi-e5 モデルの学習環境に合わせて文言を追加
+  # https://hironsan.hatenablog.com/entry/2023/07/05/073150
+  # --------------------------------------
+  if ss.current_embedding == "intfloat/multilingual-e5-large":
+    for i in range(len(texts)):
+      texts[i].page_content = "passage:" + texts[i].page_content
+
+  # vectordb の初期化
+  if ss.db is None:
+    ss = initialize_db(ss)
+
+  # db に埋め込み
+  # ss.db = Chroma.from_documents(texts, ss.embeddings)
+  ss.db.add_documents(documents=texts, embedding=ss.embeddings)
+
+  return ss
+
+def embed_ref(ss: SessionState, urls: str, fileobj: list, header_lim: int, footer_lim: int) -> (SessionState, str):
+
+  # --------------------------------------
+  # モデルロード確認
+  # --------------------------------------
+  if ss.llm is None or ss.embeddings is None:
+    status_message = "❌ LLM/Embeddingモデルが登録されていません。"
+    return ss, status_message
+
+  url_flag = "-"
+  pdf_flag = "-"
+
+  # --------------------------------------
+  # URLの読み込みとvectordb登録
+  # --------------------------------------
+
+  # URLリストの前処理（リスト化、重複削除、非URL排除）
+  urls = list({url for url in urls.split("\n") if url and "://" in url})
+
+  if urls:
+    # 登録済みURL(ss.embedded_urls)との重複を排除。登録済みリストに登録
+    urls = [url for url in urls if url not in ss.embedded_urls]
+    ss.embedded_urls.extend(urls)
+
+    # ウェブページの読み込み
+    loader = SeleniumURLLoader(urls=urls)
+    ref_documents = loader.load()
+
+    # 埋め込み処理の実行
+    ss = embedding_process(ss, ref_documents)
+
+    url_flag = "✅ 登録済"
+
+  # --------------------------------------
+  # PDFのヘッダーとフッターを除去してvectordb登録
+  # 　https://pypdf.readthedocs.io/en/stable/user/extract-text.html
+  # --------------------------------------
+
+  if fileobj is None:
+    pass
+
+  else:
+    # ファイル名リストを取得
+    pdf_paths = []
+    for path in fileobj:
+      pdf_paths.append(path.name)
+
+    # リストの初期化
+    ref_documents = []
+
+    # 各PDFファイルを読み込み
+    for pdf_path in pdf_paths:
+      pdf = PdfReader(pdf_path)
+      body = []
+
+      def visitor_body(text, cm, tm, font_dict, font_size):
+        y = tm[5]
+        if y > footer_lim and y < header_lim:  # y座標がヘッダーとフッターの間にあるかどうかを確認
+          parts.append(text)
+
+      for page in pdf.pages:
+        parts = []
+        page.extract_text(visitor_text=visitor_body)
+        body.append("".join(parts))
+
+      body = "\n".join(body)
+
+      # パスからファイル名のみを取得
+      filename = os.path.basename(pdf_path)
+      # 取得テキスト　→　LangChain ドキュメント変換
+      ref_documents.append(Document(page_content=body, metadata={"source": filename}))
+
+    # 埋め込み処理の実行
+    ss = embedding_process(ss, ref_documents)
+
+    pdf_flag = "✅ 登録済"
+
+
+  langchain.debug=True
+
+  status_message = "URL: " + url_flag + " / PDF: " + pdf_flag
+  return ss, status_message
+
+def clear_db(ss: SessionState) -> (SessionState, str):
+  if ss.db is None:
+    status_message =  "❌ 参照データが登録されていません。"
+    return ss, status_message
+
+  try:
+    ss.db.delete_collection()
+    status_message = "✅ 参照データを削除しました。"
+
+  except NameError:
+    status_message =  "❌ 参照データが登録されていません。"
+
+  return ss, status_message
+
+# ----------------------------------------------------------------------------
+# query入力 ▶ [def user] ▶ [    def bot    ] ▶ [def show_response] ▶　チャットボット画面
+#                 ⬇              ⬇ ⬆
+#          チャットボット画面    [qa_predict / conversation_predict]
+# ----------------------------------------------------------------------------
+
+def user(ss: SessionState, query) -> (SessionState, list):
+  # 会話履歴が一定数を超えた場合は、最初の履歴を削除する
+  if len(ss.dialogue) > 20:
+      ss.dialogue.pop(0)
+
+  ss.dialogue   = ss.dialogue + [(query, None)] # 会話履歴（None はボットの回答欄=空欄）
+  chat_history  = ss.dialogue
+
+  # チャット画面=chat_history
+  return ss, chat_history
+
+def bot(ss: SessionState, query, qa_flag, web_flag, summarization_mode) -> (SessionState, str):
+
+  original_query = query
+
+  if ss.llm is None:
+    response = "LLMが設定されていません。設定画面で任意のモデルを選択してください。"
+    ss.dialogue[-1] = (ss.dialogue[-1][0], response)
+    return ss, ""
+
+  elif qa_flag is True and ss.embeddings is None:
+    response = "Embeddingモデルが設定されていません。設定画面で任意のモデルを選択してください。"
+    ss.dialogue[-1] = (ss.dialogue[-1][0], response)
+
+  elif qa_flag is True and ss.db is None:
+    response = "参照データが登録されていません。"
+    ss.dialogue[-1] = (ss.dialogue[-1][0], response)
+
+  # Refine query
+  history = ss.memory.load_memory_variables({})
+  if history['chat_history'] != "":
+    # チャット履歴からクエリをリファイン
+    query = ss.query_generator({"query": query, "chat_history": history})['text']
+
+  # QA Model
+  if qa_flag is True and ss.embeddings is not None and ss.db is not None:
+    if web_flag:
+      web_query = web_search(query, ss.current_model)
+      ss = qa_predict(ss, web_query)
+      ss.memory.chat_memory.messages[-2].content = query
+    else:
+      ss = qa_predict(ss, query)      # LLMで回答を生成
+
+  # Chat Model
+  else:
+    if web_flag:
+      web_query = web_search(query, ss.current_model)
+      ss = chat_predict(ss, web_query)
+      ss.memory.chat_memory.messages[-2].content = query
+    else:
+      ss = chat_predict(ss, query)
+
+  # GPTモデル利用時はDeepLでメモリを英語化
+  ss = deepl_memory(ss)
+
+  return ss, ""                     # ssとquery欄（空欄）
+
+def chat_predict(ss: SessionState, query) -> SessionState:
+  response = ss.conversation_chain.predict(query=query)
+  ss.dialogue[-1] = (ss.dialogue[-1][0], response)
+  return ss
+
+def qa_predict(ss: SessionState, query) -> SessionState:
+
+  # Rinnaモデル向けの設定（クエリの改行コード修正）
+  if ss.current_model == "rinna/bilingual-gpt-neox-4b-instruction-sft":
+    query = query.strip().replace("\n", "<NL>")
+  else:
+    query = query.strip()
+
+  # multilingual-e5向けのクエリ文言prefix
+  if ss.current_embedding == "intfloat/multilingual-e5-large":
+    db_query_str = "query: " + query
+  else:
+    db_query_str = query
+
+  # DBから関連文書と出典を抽出
+  docs = ss.db.similarity_search(db_query_str, k=ss.similarity_search_k)
+  sources= "\n\n[Sources]\n" + '\n - '.join(list(set(doc.metadata['source'] for doc in docs if 'source' in doc.metadata)))
+
+  # Rinnaモデル向けの設定（抽出文書の改行コード修正）
+  if ss.current_model == "rinna/bilingual-gpt-neox-4b-instruction-sft":
+    for i in range(len(docs)):
+      docs[i].page_content = docs[i].page_content.strip().replace("\n", "<NL>")
+
+  # 回答の生成（最大3回の試行）
+  for _ in range(3):
+      result = ss.qa_chain({"input_documents": docs, "query": query})
+      result["output_text"] = result["output_text"].replace("<NL>", "\n").strip("...").strip("回答:").strip()
+
+      # result["output_text"]が空欄でない場合、メモリーを更新して返す
+      if result["output_text"] != "":
+        response = result["output_text"] + sources
+        ss.dialogue[-1] = (ss.dialogue[-1][0], response)
+        return ss
+      else:
+        # 空欄の場合は直近の履歴を削除してやり直し
+        ss.memory.chat_memory.messages = ss.memory.chat_memory.messages[:-2]
+
+  # 3回の試行後も空欄の場合
+  response = "3回試行しましたが、情報製生成できませんでした。"
+  if sources != "":
+    response += "参考文献の抽出には成功していますので、言語モデルを変えてお���しください。"
+
+  # ユーザーメッセージと AI メッセージの追加
+  ss.memory.chat_memory.add_user_message(query.replace("<NL>", "\n"))
+  ss.memory.chat_memory.add_ai_message(response)
+  ss.dialogue[-1] = (ss.dialogue[-1][0], response)  # 会話履歴
+  return ss
+
+# 回答を1文字ずつチャット画面に表示する
+def show_response(ss: SessionState) -> str:
+  chat_history = [list(item) for item in ss.dialogue]   # タプルをリストに変換して、メモリから会話履歴を取得
+  response = chat_history[-1][1]                        # メモリから最新の会話[-1]を取得し、チャットボットの回答[1]を退避
+  chat_history[-1][1] = ""                              # 逐次表示のため、チャットボットの回答[1]を空にする
+
+  if response is None:
+    response = "回答を生成できませんでした。"
+
+  for character in response:
+    chat_history[-1][1] += character
+    time.sleep(0.05)
+    yield chat_history
+
+with gr.Blocks() as demo:
+
+  # ユーザ別セッションメモリのインスタンス化（リロードでリセット）
+  ss = gr.State(SessionState())
+
+  # --------------------------------------
+  # API KEY をセット/クリアする関数
+  # --------------------------------------
+  def openai_api_setfn(openai_api_key) -> str:
+    if openai_api_key == "kikagaku":
+      os.environ["OPENAI_API_KEY"] = os.getenv("kikagaku_demo")
+      status_message = "✅ キカガク専用DEMOへようこそ！APIキーを設定しました"
+      return status_message
+    elif not openai_api_key or not openai_api_key.startswith("sk-") or len(openai_api_key) < 50:
+      os.environ["OPENAI_API_KEY"] = ""
+      status_message = "❌ 有効なAPIキーを入力してください"
+      return status_message
+    else:
+      os.environ["OPENAI_API_KEY"] = openai_api_key
+      status_message = "✅ APIキーを設定しました"
+      return status_message
+
+  def openai_api_clsfn(ss) -> (str, str):
+    openai_api_key = ""
+    os.environ["OPENAI_API_KEY"] = ""
+    status_message = "✅ APIキーの削除が完了しました"
+    return status_message, ""
+
+  with gr.Tabs():
+    # --------------------------------------
+    # Setting Tab
+    # --------------------------------------
+    with gr.TabItem("1. LLM設定"):
+      with gr.Row():
+        model_id = gr.Dropdown(
+          choices=[
+          'elyza/ELYZA-japanese-Llama-2-7b-fast-instruct',
+          'rinna/bilingual-gpt-neox-4b-instruction-sft',
+          'gpt-3.5-turbo',
+          ],
+          value="gpt-3.5-turbo",
+          label='LLM model',
+          interactive=True,
+        )
+      with gr.Row():
+        embedding_id = gr.Dropdown(
+          choices=[
+          'intfloat/multilingual-e5-large',
+          'sonoisa/sentence-bert-base-ja-mean-tokens-v2',
+          'oshizo/sbert-jsnli-luke-japanese-base-lite',
+          'text-embedding-ada-002',
+          "None"
+          ],
+          value="text-embedding-ada-002",
+          label = 'Embedding model',
+          interactive=True,
+        )
+      with gr.Row():
+        with gr.Column(scale=19):
+          openai_api_key = gr.Textbox(label="OpenAI API Key (Optional)", interactive=True, type="password", value="", placeholder="Your OpenAI API Key for OpenAI models.", max_lines=1)
+        with gr.Column(scale=1):
+          openai_api_set = gr.Button(value="Set API KEY", size="sm")
+          openai_api_cls = gr.Button(value="Delete API KEY", size="sm")
+
+      # with gr.Row():
+      #     reference_libs          = gr.CheckboxGroup(choices=['LangChain', 'Gradio'], label="Reference Libraries", interactive=False)
+
+      # 詳細設定（折りたたみ）
+      with gr.Accordion(label="Advanced Setting", open=False):
+        with gr.Row():
+          with gr.Column():
+            load_in_8bit          = gr.Checkbox(label="8bit Quantize (HF)", value=True, interactive=True)
+            verbose               = gr.Checkbox(label="Verbose (OpenAI, HF)", value=True, interactive=True)
+          with gr.Column():
+            temperature           = gr.Slider(label='Temperature (OpenAI, HF)', minimum=0.0, maximum=1.0, step=0.1, value=0.2, interactive=True)
+          with gr.Column():
+            similarity_search_k		=	gr.Slider(label="similarity_search_k (OpenAI, HF)", minimum=1, maximum=10, step=1, value=3, interactive=True)
+          with gr.Column():
+            summarization_mode    = gr.Radio(choices=['stuff', 'map_reduce'], label="Summarization mode", value='stuff', interactive=True)
+          with gr.Column():
+            min_length						=	gr.Slider(label="min_length (HF)", minimum=1, maximum=100, step=1, value=10, interactive=True)
+          with gr.Column():
+            max_new_tokens				=	gr.Slider(label="max_tokens(OpenAI), max_new_tokens(HF)", minimum=1, maximum=1024, step=1, value=256, interactive=True)
+          with gr.Column():
+            top_k								  =	gr.Slider(label='top_k (HF)', minimum=1, maximum=100, step=1, value=40, interactive=True)
+          with gr.Column():
+            top_p								  =	gr.Slider(label='top_p (HF)', minimum=0.01, maximum=0.99, step=0.01, value=0.92, interactive=True)
+          with gr.Column():
+            repetition_penalty		=	gr.Slider(label='repetition_penalty (HF)', minimum=0.5, maximum=2, step=0.1, value=1.2, interactive=True)
+          with gr.Column():
+            num_return_sequences	=	gr.Slider(label='num_return_sequences (HF)', minimum=1, maximum=20, step=1, value=3, interactive=True)
+
+      with gr.Row():
+        with gr.Column(scale=2):
+          config_btn = gr.Button(value="Configure")
+        with gr.Column(scale=13):
+          status_cfg = gr.Textbox(show_label=False, interactive=False, value="モデルを設定してください", container=False, max_lines=1)
+
+      # ボタン等のアクション設定
+      openai_api_set.click(openai_api_setfn, inputs=[openai_api_key], outputs=[status_cfg], show_progress="full")
+      openai_api_cls.click(openai_api_clsfn, inputs=[openai_api_key], outputs=[status_cfg, openai_api_key], show_progress="full")
+      openai_api_key.submit(openai_api_setfn, inputs=[openai_api_key], outputs=[status_cfg], show_progress="full")
+      config_btn.click(
+          fn            = load_models,
+          inputs        = [ss, model_id, embedding_id, openai_api_key, load_in_8bit, verbose, temperature, \
+                           similarity_search_k, summarization_mode, \
+                           min_length, max_new_tokens, top_k, top_p, repetition_penalty, num_return_sequences],
+          outputs       = [ss, status_cfg],
+          queue         = True,
+          show_progress = "full"
+      )
+
+    # --------------------------------------
+    # Reference Tab
+    # --------------------------------------
+    with gr.TabItem("2. References"):
+      urls = gr.TextArea(
+        max_lines = 60,
+        show_label=False,
+        info = "List any reference URLs for Q&A retrieval.",
+        placeholder = "https://blog.kikagaku.co.jp/deep-learning-transformer\nhttps://note.com/elyza/n/na405acaca130",
+        interactive=True,
+      )
+
+      with gr.Row():
+        pdf_paths  = gr.File(label="PDFs", height=150, min_width=60, scale=7, file_types=[".pdf"], file_count="multiple", interactive=True)
+        header_lim = gr.Number(label="Header (pt)", step=1, value=792, precision=0, min_width=70, scale=1, interactive=True)
+        footer_lim = gr.Number(label="Footer (pt)", step=1, value=0, precision=0, min_width=70, scale=1, interactive=True)
+        pdf_ref = gr.Textbox(show_label=False, value="A4 Size:\n(下)0-792pt(上)\n  *28.35pt/cm", container=False, scale=1, interactive=False)
+
+      with gr.Row():
+        ref_set_btn = gr.Button(value="コンテンツ登録", scale=1)
+        ref_clear_btn = gr.Button(value="登録データ削除", scale=1)
+        status_ref = gr.Textbox(show_label=False, interactive=False, value="参照データ未登録", container=False, max_lines=1, scale=18)
+
+      ref_set_btn.click(fn=embed_ref, inputs=[ss, urls, pdf_paths, header_lim, footer_lim], outputs=[ss, status_ref], queue=True, show_progress="full")
+      ref_clear_btn.click(fn=clear_db, inputs=[ss], outputs=[ss, status_ref], show_progress="full")
+
+    # --------------------------------------
+    # Chatbot Tab
+    # --------------------------------------
+    with gr.TabItem("3. Q&A Chat"):
+      chat_history = gr.Chatbot([], elem_id="chatbot", avatar_images=["bear.png", "penguin.png"],)
+      with gr.Row():
+        with gr.Column(scale=95):
+          query = gr.Textbox(
+            show_label=False,
+            placeholder="Send a message with [Shift]+[Enter] key.",
+            lines=4,
+            container=False,
+            autofocus=True,
+            interactive=True,
+          )
+        with gr.Column(scale=5):
+          with gr.Row():
+            qa_flag = gr.Checkbox(label="QA mode", value=True, min_width=60, interactive=True)
+            web_flag = gr.Checkbox(label="Web Search", value=False, min_width=60, interactive=True)
+          with gr.Row():
+            query_send_btn = gr.Button(value="▶")
+
+      # gr.Examples(["機械学習について説明してください"], inputs=[query])
+      query.submit(
+        user, [ss, query], [ss, chat_history]
+      ).then(
+        bot, [ss, query, qa_flag, web_flag, summarization_mode], [ss, query]
+      ).then(
+        show_response, [ss], [chat_history]
+      )
+
+      query_send_btn.click(
+        user, [ss, query], [ss, chat_history]
+      ).then(
+        bot, [ss, query, qa_flag, web_flag, summarization_mode], [ss, query]
+      ).then(
+        show_response, [ss], [chat_history]
+      )
+
+if __name__ == "__main__":
+    demo.queue(concurrency_count=5)
+    demo.launch(debug=True)

packages.txt

@@ -0,0 +1 @@
+chromium-driver

requirements.txt

@@ -0,0 +1,23 @@
+accelerate==0.22.0
+beautifulsoup4==4.11.2
+bitsandbytes==0.41.1
+transformers==4.30.0
+sentence-transformers==2.2.2
+sentencepiece==0.1.99
+langchain==0.0.281
+xformers==0.0.21
+chromadb==0.4.8
+gradio==3.42.0
+gradio_client==0.5.0
+openai==0.28.0
+tiktoken==0.4.0
+fugashi==1.3.0
+ipadic==1.0.0
+unstructured==0.10.12
+selenium==4.12.0
+pypdf==3.15.5
+Cython==0.29.36
+numpy==1.23.5
+pandas==1.5.3
+chromedriver-autoinstaller
+chromedriver-binary