vectostoreindex

.gitignore

@@ -1,5 +1,3 @@
 /myenv
 __pycache__/
 appcompleta.py
-appLlama.py
-interface.py

app-last.py

@@ -0,0 +1,244 @@
+import os
+import spaces
+from threading import Thread
+from typing import Iterator
+from backend2 import load_documents, prepare_documents, get_context_sources
+import gradio as gr
+import torch
+from transformers import AutoModelForCausalLM, GemmaTokenizerFast, TextIteratorStreamer
+from huggingface_hub import login
+from llama_index.core import Settings, VectorStoreIndex, SimpleDirectoryReader, ChatPromptTemplate, PromptTemplate, load_index_from_storage, StorageContext
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.embeddings.instructor import InstructorEmbedding
+
+
+huggingface_token = os.getenv("HUGGINGFACE_TOKEN")
+login(huggingface_token)
+
+DESCRIPTION = """\
+# La Chatbot degli Osservatori
+"""
+MAX_MAX_NEW_TOKENS = 2048
+DEFAULT_MAX_NEW_TOKENS = 1024
+os.environ["MAX_INPUT_TOKEN_LENGTH"] = "4096" #"8192"
+MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH"))
+
+
+# Force usage of CPU
+#device = torch.device("cpu")
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+model_id = "google/gemma-2-2b-it"
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    device_map="auto",
+    torch_dtype= torch.bfloat16 #torch.float16 if torch.cuda.is_available() else torch.float32,
+)
+tokenizer = GemmaTokenizerFast.from_pretrained(model_id)
+#tokenizer = AutoTokenizer.from_pretrained(model_id)
+tokenizer.use_default_system_prompt = False
+model.config.sliding_window = 4096
+#model = model.to(device)
+model.eval()
+
+###------####
+# rag
+documents_paths = {
+    'blockchain': 'documents/blockchain',
+    'metaverse': 'documents/metaverso',
+    'payment': 'documents/payment'
+}
+
+global session_state
+session_state = {"index": False,
+                 "documents_loaded": False, 
+                 "document_db": None, 
+                 "original_message": None, 
+                 "clarification": False}
+
+INSTRUCTION_1 = 'In italiano, chiedi sempre se la domanda si riferisce agli "Osservatori Blockchain", "Osservatori Payment" oppure "Osservatori Metaverse".'
+INSTRUCTION_2 = 'Sei un assistente che risponde sempre in italiano alle domande basandosi solo sulle informazioni fornite nel contesto che ti darò. Se non trovi informazioni, rispondi "Puoi chiedere maggiori informazioni all\'ufficio di riferimento.". Se invece la domanda è completamente fuori contesto, non rispondere e rammenta il topic del contesto'
+
+
+"""# Reading documents from disk
+docs = SimpleDirectoryReader(input_files=["data/blockchainprova.txt"]).load_data()
+            # Splitting the document into chunks with
+            # predefined size and overlap
+parser = SentenceSplitter.from_defaults(
+                chunk_size=256, chunk_overlap=64, paragraph_separator="\n\n"
+            )
+nodes = parser.get_nodes_from_documents(docs)"""
+
+
+@spaces.GPU()
+def generate(
+    message: str,
+    chat_history: list[tuple[str, str]],
+    max_new_tokens: int = 1024,
+    temperature: float = 0.6,
+    top_p: float = 0.9,
+    top_k: int = 50,
+    repetition_penalty: float = 1.2,
+) -> Iterator[str]:
+    
+    
+    global matched_path
+    
+    conversation = []
+    for user, assistant in chat_history:
+        conversation.extend(
+            [
+                {"role": "user", "content": user},
+                {"role": "assistant", "content": assistant},
+            ]
+        )
+    
+    
+    if not session_state["index"]:
+        
+        matched_path = None
+        words = message.lower()
+        for key, path in documents_paths.items():
+            if key in words:
+                matched_path = path
+                break
+        if matched_path:
+            documents = load_documents(matched_path)
+            DB = prepare_documents(documents)
+            context, sources = get_context_sources(message, DB)
+            print("*** sources ***", sources)
+            gr.Info("doc preparati con ", sources)
+            
+            conversation.append({"role": "user", "content": f'Contesto: {context}\n\n Domanda: {message}. Rispondi in italiano'})
+
+            ######
+            
+            """index = VectorStoreIndex(nodes)
+            # get retriver
+            retriever = index.as_retriever(similarity_top_k=3)
+            relevant_chunks = retriever.retrieve(message)
+            print(f"Found: {len(relevant_chunks)} relevant chunks")
+            for idx, chunk in enumerate(relevant_chunks):
+                
+                info_message += f"{idx + 1}) {chunk.text[:64]}...\n"
+                print(info_message)
+                gr.Info(info_message)"""
+            
+            session_state["index"] = True
+            
+        else: ## CHIEDI CHIARIMENTO
+          
+            conversation.append({"role": "user", "content": f"Domanda: {message} . Comando: {INSTRUCTION_1}" })
+            gr.Info("richiesta di chiarimento")
+            print("******** CONV1 ", conversation)
+
+            
+                  
+    else:
+        
+        documents = load_documents(matched_path)
+        DB = prepare_documents(documents)
+        context, sources = get_context_sources(message, DB)
+        gr.Info("contesto già indicizzato")
+        conversation.append({"role": "user", "content": f"{INSTRUCTION_2}"})
+        conversation.append({"role": "assistant", "content": "Ok."})  
+        conversation.append({"role": "user", "content": f'Contesto: {context}\n\n Domanda: {message}. Rispondi in italiano'})
+        
+        print("******** CONV2 ", conversation)
+        
+    
+
+    # Iterate model output
+    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt")
+    if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
+        input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
+        gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
+    input_ids = input_ids.to(model.device)
+
+    streamer = TextIteratorStreamer(tokenizer, timeout=None, skip_prompt=True, skip_special_tokens=True)
+    generate_kwargs = dict(
+        {"input_ids": input_ids},
+        streamer=streamer,
+        max_new_tokens=max_new_tokens,
+        do_sample=True,
+        top_p=top_p,
+        top_k=top_k,
+        temperature=temperature,
+        num_beams=1,
+        repetition_penalty=repetition_penalty,
+    )
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
+    
+
+    outputs = []
+    for text in streamer:
+        outputs.append(text)
+        yield "".join(outputs)
+    
+    if session_state["documents_loaded"]:
+        outputs.append(f"Fonti utilizzate: {sources}")
+        yield "".join(outputs) 
+        
+    #sources = []    
+    print("debug - CHATHISTORY", chat_history)    
+
+chat_interface = gr.ChatInterface(
+    fn=generate,
+    additional_inputs=[
+        gr.Slider(
+            label="Max new tokens",
+            minimum=1,
+            maximum=MAX_MAX_NEW_TOKENS,
+            step=1,
+            value=DEFAULT_MAX_NEW_TOKENS,
+        ),
+        gr.Slider(
+            label="Temperature",
+            minimum=0.1,
+            maximum=4.0,
+            step=0.1,
+            value=0.6,
+        ),
+        gr.Slider(
+            label="Top-p (nucleus sampling)",
+            minimum=0.05,
+            maximum=1.0,
+            step=0.05,
+            value=0.9,
+        ),
+        gr.Slider(
+            label="Top-k",
+            minimum=1,
+            maximum=1000,
+            step=1,
+            value=50,
+        ),
+        gr.Slider(
+            label="Repetition penalty",
+            minimum=1.0,
+            maximum=2.0,
+            step=0.05,
+            value=1.2,
+        ),
+    ],
+    stop_btn=None,
+    examples=[
+        ["Ciao, in cosa puoi aiutarmi?"],
+        ["Ciao, in cosa consiste un piatto di spaghetti?"],
+        ["Ciao, quali sono le aziende che hanno iniziato ad integrare le stablecoins? Fammi un breve sommario."],
+        ["Spiegami la differenza tra mondi virtuali pubblici o privati"],
+        ["Trovami un esempio di progetto B2B"],
+        ["Quali sono le regole europee sui bonifici istantanei?"],
+    ],
+    cache_examples=False,
+)
+
+with gr.Blocks(css=".gradio-container {background-color: #B9D9EB}", fill_height=True) as demo:
+    gr.Markdown(DESCRIPTION, elem_classes="centered")
+    chat_interface.render()
+
+if __name__ == "__main__":
+    #demo.queue(max_size=20).launch()
+    demo.launch()

app.py

@@ -1,244 +1,29 @@
-import os
-import spaces
-from threading import Thread
-from typing import Iterator
-from backend2 import load_documents, prepare_documents, get_context_sources
+from backend import handle_query
 import gradio as gr
-import torch
-from transformers import AutoModelForCausalLM, GemmaTokenizerFast, TextIteratorStreamer
-from huggingface_hub import login
-from llama_index.core import Settings, VectorStoreIndex, SimpleDirectoryReader, ChatPromptTemplate, PromptTemplate, load_index_from_storage, StorageContext
-from llama_index.core.node_parser import SentenceSplitter
-from llama_index.embeddings.instructor import InstructorEmbedding
 
 
-huggingface_token = os.getenv("HUGGINGFACE_TOKEN")
-login(huggingface_token)
-
 DESCRIPTION = """\
-# La Chatbot degli Osservatori
-"""
-MAX_MAX_NEW_TOKENS = 2048
-DEFAULT_MAX_NEW_TOKENS = 1024
-os.environ["MAX_INPUT_TOKEN_LENGTH"] = "4096" #"8192"
-MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH"))
-
-
-# Force usage of CPU
-#device = torch.device("cpu")
-
-device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
-
-model_id = "google/gemma-2-2b-it"
-model = AutoModelForCausalLM.from_pretrained(
-    model_id,
-    device_map="auto",
-    torch_dtype= torch.bfloat16 #torch.float16 if torch.cuda.is_available() else torch.float32,
-)
-tokenizer = GemmaTokenizerFast.from_pretrained(model_id)
-#tokenizer = AutoTokenizer.from_pretrained(model_id)
-tokenizer.use_default_system_prompt = False
-model.config.sliding_window = 4096
-#model = model.to(device)
-model.eval()
+# <div style="text-align: center;">Odi, l'assistente ricercatore degli Osservatori</div>
 
-###------####
-# rag
-documents_paths = {
-    'blockchain': 'documents/blockchain',
-    'metaverse': 'documents/metaverso',
-    'payment': 'documents/payment'
-}
 
-global session_state
-session_state = {"index": False,
-                 "documents_loaded": False, 
-                 "document_db": None, 
-                 "original_message": None, 
-                 "clarification": False}
-
-INSTRUCTION_1 = 'In italiano, chiedi sempre se la domanda si riferisce agli "Osservatori Blockchain", "Osservatori Payment" oppure "Osservatori Metaverse".'
-INSTRUCTION_2 = 'Sei un assistente che risponde sempre in italiano alle domande basandosi solo sulle informazioni fornite nel contesto che ti darò. Se non trovi informazioni, rispondi "Puoi chiedere maggiori informazioni all\'ufficio di riferimento.". Se invece la domanda è completamente fuori contesto, non rispondere e rammenta il topic del contesto'
-
-
-"""# Reading documents from disk
-docs = SimpleDirectoryReader(input_files=["data/blockchainprova.txt"]).load_data()
-            # Splitting the document into chunks with
-            # predefined size and overlap
-parser = SentenceSplitter.from_defaults(
-                chunk_size=256, chunk_overlap=64, paragraph_separator="\n\n"
-            )
-nodes = parser.get_nodes_from_documents(docs)"""
+👉 Retrieval-Augmented Generation - Ask me anything about the research carried out at the Osservatori.
+"""
 
 
-@spaces.GPU()
-def generate(
-    message: str,
-    chat_history: list[tuple[str, str]],
-    max_new_tokens: int = 1024,
-    temperature: float = 0.6,
-    top_p: float = 0.9,
-    top_k: int = 50,
-    repetition_penalty: float = 1.2,
-) -> Iterator[str]:
-    
-    
-    global matched_path
-    
-    conversation = []
-    for user, assistant in chat_history:
-        conversation.extend(
-            [
-                {"role": "user", "content": user},
-                {"role": "assistant", "content": assistant},
-            ]
+chat_interface =gr.ChatInterface(
+        fn=handle_query, 
+        chatbot=gr.Chatbot(height=500),
+        textbox=gr.Textbox(placeholder="Chiedimi qualasiasi cosa relativa agli Osservatori", container=False, scale=7),
+        #examples=[["Ciao, in cosa puoi aiutarmi?"],["Dimmi i risultati e le modalità di conduzione del censimento per favore"]]
         )
-    
-    
-    if not session_state["index"]:
-        
-        matched_path = None
-        words = message.lower()
-        for key, path in documents_paths.items():
-            if key in words:
-                matched_path = path
-                break
-        if matched_path:
-            documents = load_documents(matched_path)
-            DB = prepare_documents(documents)
-            context, sources = get_context_sources(message, DB)
-            print("*** sources ***", sources)
-            gr.Info("doc preparati con ", sources)
-            
-            conversation.append({"role": "user", "content": f'Contesto: {context}\n\n Domanda: {message}. Rispondi in italiano'})
 
-            ######
-            
-            """index = VectorStoreIndex(nodes)
-            # get retriver
-            retriever = index.as_retriever(similarity_top_k=3)
-            relevant_chunks = retriever.retrieve(message)
-            print(f"Found: {len(relevant_chunks)} relevant chunks")
-            for idx, chunk in enumerate(relevant_chunks):
-                
-                info_message += f"{idx + 1}) {chunk.text[:64]}...\n"
-                print(info_message)
-                gr.Info(info_message)"""
-            
-            session_state["index"] = True
-            
-        else: ## CHIEDI CHIARIMENTO
-          
-            conversation.append({"role": "user", "content": f"Domanda: {message} . Comando: {INSTRUCTION_1}" })
-            gr.Info("richiesta di chiarimento")
-            print("******** CONV1 ", conversation)
-
-            
-                  
-    else:
-        
-        documents = load_documents(matched_path)
-        DB = prepare_documents(documents)
-        context, sources = get_context_sources(message, DB)
-        gr.Info("contesto già indicizzato")
-        conversation.append({"role": "user", "content": f"{INSTRUCTION_2}"})
-        conversation.append({"role": "assistant", "content": "Ok."})  
-        conversation.append({"role": "user", "content": f'Contesto: {context}\n\n Domanda: {message}. Rispondi in italiano'})
-        
-        print("******** CONV2 ", conversation)
-        
     
-
-    # Iterate model output
-    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt")
-    if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
-        input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
-        gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
-    input_ids = input_ids.to(model.device)
-
-    streamer = TextIteratorStreamer(tokenizer, timeout=None, skip_prompt=True, skip_special_tokens=True)
-    generate_kwargs = dict(
-        {"input_ids": input_ids},
-        streamer=streamer,
-        max_new_tokens=max_new_tokens,
-        do_sample=True,
-        top_p=top_p,
-        top_k=top_k,
-        temperature=temperature,
-        num_beams=1,
-        repetition_penalty=repetition_penalty,
-    )
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
-    t.start()
+with gr.Blocks(css=".gradio-container {background-color: #B9D9EB}") as demo:
+    gr.Markdown(DESCRIPTION)
+    #gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
+    chat_interface.render() 
     
-
-    outputs = []
-    for text in streamer:
-        outputs.append(text)
-        yield "".join(outputs)
-    
-    if session_state["documents_loaded"]:
-        outputs.append(f"Fonti utilizzate: {sources}")
-        yield "".join(outputs) 
-        
-    #sources = []    
-    print("debug - CHATHISTORY", chat_history)    
-
-chat_interface = gr.ChatInterface(
-    fn=generate,
-    additional_inputs=[
-        gr.Slider(
-            label="Max new tokens",
-            minimum=1,
-            maximum=MAX_MAX_NEW_TOKENS,
-            step=1,
-            value=DEFAULT_MAX_NEW_TOKENS,
-        ),
-        gr.Slider(
-            label="Temperature",
-            minimum=0.1,
-            maximum=4.0,
-            step=0.1,
-            value=0.6,
-        ),
-        gr.Slider(
-            label="Top-p (nucleus sampling)",
-            minimum=0.05,
-            maximum=1.0,
-            step=0.05,
-            value=0.9,
-        ),
-        gr.Slider(
-            label="Top-k",
-            minimum=1,
-            maximum=1000,
-            step=1,
-            value=50,
-        ),
-        gr.Slider(
-            label="Repetition penalty",
-            minimum=1.0,
-            maximum=2.0,
-            step=0.05,
-            value=1.2,
-        ),
-    ],
-    stop_btn=None,
-    examples=[
-        ["Ciao, in cosa puoi aiutarmi?"],
-        ["Ciao, in cosa consiste un piatto di spaghetti?"],
-        ["Ciao, quali sono le aziende che hanno iniziato ad integrare le stablecoins? Fammi un breve sommario."],
-        ["Spiegami la differenza tra mondi virtuali pubblici o privati"],
-        ["Trovami un esempio di progetto B2B"],
-        ["Quali sono le regole europee sui bonifici istantanei?"],
-    ],
-    cache_examples=False,
-)
-
-with gr.Blocks(css=".gradio-container {background-color: #B9D9EB}", fill_height=True) as demo:
-    gr.Markdown(DESCRIPTION, elem_classes="centered")
-    chat_interface.render()
-
 if __name__ == "__main__":
-    #demo.queue(max_size=20).launch()
-    demo.launch()
+    #progress = gr.Progress(track_tqdm=True)
+    demo.launch()
+       

backend.py

@@ -65,7 +65,7 @@ llm = HuggingFaceLLM(
     model_kwargs={"torch_dtype": torch.float16, "load_in_8bit": True},
 )
 
-Settings.llm = GemmaLLMInterface()
+#Settings.llm = GemmaLLMInterface()
 Settings.llm = llm
 
 

interface.py

@@ -0,0 +1,105 @@
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from llama_index.core.llms import  CustomLLM, LLMMetadata, CompletionResponse, CompletionResponseGen
+from llama_index.core.llms.callbacks import llm_completion_callback
+from typing import Any, Iterator
+import torch
+from transformers import TextIteratorStreamer
+from threading import Thread
+from pydantic import Field, field_validator
+import keras
+import keras_nlp
+
+# for transformers 2 (__setattr__ is used to bypass Pydantic check )
+"""class GemmaLLMInterface(CustomLLM):
+    def __init__(self, model_id: str = "google/gemma-2-2b-it", **kwargs):
+        super().__init__(**kwargs)
+        object.__setattr__(self, "model_id", model_id)  
+        model = AutoModelForCausalLM.from_pretrained(
+            model_id,
+            device_map="auto",
+            torch_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
+        )
+        tokenizer = AutoTokenizer.from_pretrained(model_id)
+        object.__setattr__(self, "model", model)  
+        object.__setattr__(self, "tokenizer", tokenizer) 
+        object.__setattr__(self, "context_window", 8192)
+        object.__setattr__(self, "num_output", 2048)
+
+    def _format_prompt(self, message: str) -> str:
+        return (
+            f"<start_of_turn>user\n{message}<end_of_turn>\n"
+            f"<start_of_turn>model\n"
+        )
+
+    @property
+    def metadata(self) -> LLMMetadata:
+        return LLMMetadata(
+            context_window=self.context_window,
+            num_output=self.num_output,
+            model_name=self.model_id, 
+        )
+        
+        
+    @llm_completion_callback()
+    def complete(self, prompt: str, **kwargs: Any) -> CompletionResponse:
+        prompt = self._format_prompt(prompt)
+        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+        outputs = self.model.generate(**inputs, max_new_tokens=self.num_output)
+        response = self.tokenizer.decode(outputs[0], skip_special_tokens=True).strip()
+        response = response[len(prompt):].strip()
+        return CompletionResponse(text=response if response else "No response generated.")
+
+    @llm_completion_callback()
+    def stream_complete(self, prompt: str, **kwargs: Any) -> CompletionResponseGen:
+        #prompt = self._format_prompt(prompt)
+        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+        
+        streamer = TextIteratorStreamer(self.tokenizer, skip_special_tokens=True)
+        generation_kwargs = dict(inputs, max_new_tokens=self.num_output, streamer=streamer)
+        
+        thread = Thread(target=self.model.generate, kwargs=generation_kwargs)
+        thread.start()
+
+        streamed_response = ""
+        for new_text in streamer:
+            if new_text:
+                streamed_response += new_text
+                yield CompletionResponse(text=streamed_response, delta=new_text)
+        
+        if not streamed_response:
+            yield CompletionResponse(text="No response generated.", delta="No response generated.")"""
+
+# for Keras
+class GemmaLLMInterface(CustomLLM):
+    model: keras_nlp.models.GemmaCausalLM = None
+    context_window: int = 8192
+    num_output: int = 2048
+    model_name: str = "gemma_2"
+
+    def _format_prompt(self, message: str) -> str:
+        return (
+            f"<start_of_turn>user\n{message}<end_of_turn>\n" f"<start_of_turn>model\n"
+        )
+
+    @property
+    def metadata(self) -> LLMMetadata:
+        """Get LLM metadata."""
+        return LLMMetadata(
+            context_window=self.context_window,
+            num_output=self.num_output,
+            model_name=self.model_name,
+        )
+
+    @llm_completion_callback()
+    def complete(self, prompt: str, **kwargs: Any) -> CompletionResponse:
+        prompt = self._format_prompt(prompt)
+        raw_response = self.model.generate(prompt, max_length=self.num_output)
+        response = raw_response[len(prompt) :]
+        return CompletionResponse(text=response)
+
+    @llm_completion_callback()
+    def stream_complete(self, prompt: str, **kwargs: Any) -> CompletionResponseGen:
+        response = self.complete(prompt).text
+        for token in response:
+            response += token
+            yield CompletionResponse(text=response, delta=token)