Update app.py

app.py

@@ -1,9 +1,8 @@
-import json
 import gradio as gr
 import gspread
 from oauth2client.service_account import ServiceAccountCredentials
 from llama_cpp import Llama
-from llama_index.core import VectorStoreIndex, Settings
+from llama_index.core import SimpleDirectoryReader, VectorStoreIndex, Settings
 from llama_index.core.node_parser import SentenceSplitter
 from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 from llama_index.llms.llama_cpp import LlamaCPP
@@ -12,9 +11,9 @@ from llama_index.core.llms import ChatMessage
 from llama_index.core.chat_engine.condense_plus_context import CondensePlusContextChatEngine
 from llama_index.core.schema import Document
 
-# ===================================
-# 1️⃣ Fungsi Membaca Data Google Spreadsheet
-# ===================================
+# ==============================
+# 1️⃣ Fungsi Membaca Data Google Spreadsheet (Cuti, Lembur, Absen, Target)
+# ==============================
 def read_google_sheets():
     try:
         scope = ["https://www.googleapis.com/auth/spreadsheets", "https://www.googleapis.com/auth/drive"]
@@ -22,7 +21,7 @@ def read_google_sheets():
         client = gspread.authorize(creds)
         
         SPREADSHEET_ID = "1e_cNMhwF-QYpyYUpqQh-XCw-OdhWS6EuYsoBUsVtdNg"
-        sheet_names = ["datatarget", "datacuti", "dataabsen", "datalembur", "pkb"]
+        sheet_names = ["datatarget", "datacuti", "dataabsen", "datalembur"]
 
         all_data = []
         spreadsheet = client.open_by_key(SPREADSHEET_ID)
@@ -45,9 +44,31 @@ def read_google_sheets():
     except Exception as e:
         return f"❌ ERROR: {str(e)}"
 
-# ===================================
-# 2️⃣ Inisialisasi Model Llama
-# ===================================
+# ==============================
+# 2️⃣ Fungsi Membaca Data dari File TXT (PKB dan Dokumen Lainnya)
+# ==============================
+def read_txt_documents():
+    documents = SimpleDirectoryReader(input_files=[
+        "bahandokumen/K3.txt", "bahandokumen/bonus.txt", "bahandokumen/absensi.txt", 
+        "bahandokumen/cuti.txt", "bahandokumen/disiplinkerja.txt", "bahandokumen/fasilitas&bantuan.txt", 
+        "bahandokumen/fasilitaskerja.txt", "bahandokumen/hak.txt", "bahandokumen/hubunganpengusaha&serikat.txt", 
+        "bahandokumen/istilah.txt", "bahandokumen/jaminanserikat.txt", "bahandokumen/jamkes.txt", 
+        "bahandokumen/jamsos.txt", "bahandokumen/keluhkesah.txt", "bahandokumen/kenaikanupah.txt", 
+        "bahandokumen/kewajiban.txt", "bahandokumen/kompensasi.txt", "bahandokumen/larangan.txt", 
+        "bahandokumen/lembur.txt", "bahandokumen/luaskesepakatan.txt", "bahandokumen/mogok.txt", 
+        "bahandokumen/pelanggaran&sanksi.txt", "bahandokumen/pendidikan.txt", "bahandokumen/pengangkatan.txt", 
+        "bahandokumen/penilaian&promosi.txt", "bahandokumen/pensiun.txt", "bahandokumen/perjadin.txt", 
+        "bahandokumen/pesangon.txt", "bahandokumen/phk.txt", "bahandokumen/pihak.txt", "bahandokumen/pkb.txt", 
+        "bahandokumen/resign.txt", "bahandokumen/sanksi.txt", "bahandokumen/shift.txt", "bahandokumen/syaratkerja.txt", 
+        "bahandokumen/sisacuti.txt", "bahandokumen/target.txt", "bahandokumen/tatacara.txt", "bahandokumen/tka.txt", 
+        "bahandokumen/tunjangan.txt", "bahandokumen/uangpisah.txt", "bahandokumen/upah.txt", "bahandokumen/upahlembur.txt", 
+        "bahandokumen/waktukerja.txt"
+    ]).load_data()
+    return documents
+
+# ==============================
+# 3️⃣ Inisialisasi Model Llama
+# ==============================
 def initialize_llama_model():
     model_path = hf_hub_download(
         repo_id="TheBLoke/zephyr-7b-beta-GGUF",
@@ -56,20 +77,23 @@ def initialize_llama_model():
     )
     return model_path
 
-# ===================================
-# 3️⃣ Inisialisasi Pengaturan Model
-# ===================================
+# ==============================
+# 4️⃣ Inisialisasi Pengaturan Model
+# ==============================
 def initialize_settings(model_path):
     Settings.llm = LlamaCPP(model_path=model_path, temperature=0.7)
 
-# ===================================
-# 4️⃣ Inisialisasi Index & Chat Engine
-# ===================================
+# ==============================
+# 5️⃣ Inisialisasi Index
+# ==============================
 def initialize_index():
     text_data = read_google_sheets()
     document = Document(text=text_data)
-    parser = SentenceSplitter(chunk_size=100, chunk_overlap=30)
-    nodes = parser.get_nodes_from_documents([document])
+    txt_documents = read_txt_documents()
+    
+    all_documents = [document] + txt_documents  # Gabungkan dokumen CSV dan TXT
+    parser = SentenceSplitter(chunk_size=150, chunk_overlap=10)
+    nodes = parser.get_nodes_from_documents(all_documents)
     
     embedding = HuggingFaceEmbedding("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
     Settings.embed_model = embedding
@@ -77,26 +101,17 @@ def initialize_index():
     index = VectorStoreIndex(nodes)
     return index
 
+# ==============================
+# 6️⃣ Inisialisasi Chat Engine
+# ==============================
 def initialize_chat_engine(index):
     retriever = index.as_retriever(similarity_top_k=3)
-    chat_engine = CondensePlusContextChatEngine.from_defaults(
-        retriever=retriever,
-        verbose=False  # ❌ Hapus verbose agar tidak ada referensi dokumen
-    )
+    chat_engine = CondensePlusContextChatEngine.from_defaults(retriever=retriever, verbose=False)
     return chat_engine
 
-# ===================================
-# 5️⃣ Fungsi untuk Merapikan Jawaban Chatbot
-# ===================================
-def clean_response(response):
-    text = "".join(response.response_gen)  # Gabungkan teks yang dihasilkan
-    text = text.replace("\n\n", "\n").strip()  # Hilangkan newline berlebihan
-    text = text.replace("user:", "").replace("jawaban:", "").replace("assistant:", "").strip()
-    return text
-
-# ===================================
-# 6️⃣ Fungsi untuk Menghasilkan Respons Chatbot
-# ===================================
+# ==============================
+# 7️⃣ Fungsi untuk Menjawab Chat
+# ==============================
 def generate_response(message, history, chat_engine):
     if history is None:
         history = []
@@ -119,24 +134,20 @@ def generate_response(message, history, chat_engine):
     history.append((message, cleaned_text))  # 🔹 Pastikan hanya teks yang masuk ke history
     return cleaned_text
 
-# ===================================
-# 7️⃣ Fungsi Utama untuk Menjalankan Aplikasi
-# ===================================
+# ==============================
+# 8️⃣ Fungsi Utama Menjalankan Aplikasi
+# ==============================
 def main():
     model_path = initialize_llama_model()
     initialize_settings(model_path)
-
+    
     index = initialize_index()
     chat_engine = initialize_chat_engine(index)
-
+    
     def chatbot_response(message, history):
         return generate_response(message, history, chat_engine)
-
-    gr.Interface(
-        fn=chatbot_response,
-        inputs=["text"],
-        outputs=["text"],
-    ).launch()
+    
+    gr.Interface(fn=chatbot_response, inputs="text", outputs="text").launch()
 
 if __name__ == "__main__":
     main()