rameshmoorthy commited on
Commit
4333093
1 Parent(s): 09a6682

Update backend/query_llm.py

Browse files
Files changed (1) hide show
  1. backend/query_llm.py +11 -10
backend/query_llm.py CHANGED
@@ -9,8 +9,9 @@ from typing import Any, Dict, Generator, List
9
  from huggingface_hub import InferenceClient
10
  from transformers import AutoTokenizer
11
 
12
- #tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1")
13
- tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-Instruct-v0.1")
 
14
  temperature = 0.5
15
  top_p = 0.7
16
  repetition_penalty = 1.2
@@ -18,16 +19,16 @@ repetition_penalty = 1.2
18
  OPENAI_KEY = getenv("OPENAI_API_KEY")
19
  HF_TOKEN = getenv("HUGGING_FACE_HUB_TOKEN")
20
 
21
- #hf_client = InferenceClient(
22
- # "mistralai/Mistral-7B-Instruct-v0.1",
23
- # token=HF_TOKEN
24
- # )
25
 
26
 
27
- hf_client = InferenceClient(
28
- "mistralai/Mixtral-8x7B-Instruct-v0.1",
29
- token=HF_TOKEN
30
- )
31
  def format_prompt(message: str, api_kind: str):
32
  """
33
  Formats the given message using a chat template.
 
9
  from huggingface_hub import InferenceClient
10
  from transformers import AutoTokenizer
11
 
12
+ tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1")
13
+ #tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-Instruct-v0.1")
14
+ #tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x22B-Instruct-v0.1")
15
  temperature = 0.5
16
  top_p = 0.7
17
  repetition_penalty = 1.2
 
19
  OPENAI_KEY = getenv("OPENAI_API_KEY")
20
  HF_TOKEN = getenv("HUGGING_FACE_HUB_TOKEN")
21
 
22
+ hf_client = InferenceClient(
23
+ "mistralai/Mistral-7B-Instruct-v0.1",
24
+ token=HF_TOKEN
25
+ )
26
 
27
 
28
+ # hf_client = InferenceClient(
29
+ # "mistralai/Mixtral-8x7B-Instruct-v0.1",
30
+ # token=HF_TOKEN
31
+ # )
32
  def format_prompt(message: str, api_kind: str):
33
  """
34
  Formats the given message using a chat template.