switched to hf_hub_download

.gitignore

@@ -1 +1,2 @@
-/myenv
+/myenv
+__pycache__/

backend.py

@@ -8,6 +8,27 @@ import gradio as gr
 from llama_index.core import ChatPromptTemplate
 from llama_index.core import Settings, VectorStoreIndex, SimpleDirectoryReader, PromptTemplate, load_index_from_storage
 from llama_index.core.node_parser import SentenceSplitter
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
+
+
+huggingface_token = os.getenv("HUGGINGFACE_TOKEN")
+
+
+hf_hub_download(
+    repo_id="google/gemma-2-2b-it-GGUF",
+    filename="2b_it_v2.gguf",
+    local_dir="./models",
+    token=huggingface_token
+)
+
+llm = Llama(
+            model_path=f"models/{"2b_it_v2.gguf"}",
+            #flash_attn=True,
+            #_gpu_layers=81,
+            n_batch=1024,
+            n_ctx=8192,
+        )
 
 
 model_id = "google/gemma-2-2b-it"
@@ -19,7 +40,8 @@ model = AutoModelForCausalLM.from_pretrained(
 )
 # what models will be used by LlamaIndex:
 Settings.embed_model = InstructorEmbedding(model_name="hkunlp/instructor-base")
-Settings.llm  = GemmaLLMInterface(model=model, tokenizer=tokenizer)
+#Settings.llm  = GemmaLLMInterface(model=model, tokenizer=tokenizer)
+Settings.llm  = llm
 
 
 ############################---------------------------------

requirements.txt

@@ -7,4 +7,6 @@ sentence-transformers==2.2.2
 llama-index-readers-web
 llama-index-readers-file
 gradio
-transformers
+transformers
+llama-cpp-agent>=0.2.25
+setuptools